22  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.

    22  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.

    34 /* interface connection interfaces */

    35 static void iscsi_client_notify_task(void *cn_task_void);

    35 static iscsi_status_t iscsi_conn_state_free(iscsi_conn_t *icp,

    36 

    36     iscsi_conn_event_t event);

    37 static void iscsi_conn_state_in_login(iscsi_conn_t *icp,

    38     iscsi_conn_event_t event);

    39 static void iscsi_conn_state_logged_in(iscsi_conn_t *icp,

    40     iscsi_conn_event_t event);

    41 static void iscsi_conn_state_in_logout(iscsi_conn_t *icp,

    42     iscsi_conn_event_t event);

    43 static void iscsi_conn_state_failed(iscsi_conn_t *icp,

    44     iscsi_conn_event_t event);

    45 static void iscsi_conn_state_polling(iscsi_conn_t *icp,

    46     iscsi_conn_event_t event);

    47 static char *iscsi_conn_event_str(iscsi_conn_event_t event);

   116 	/* Creation of the receive thread */

   105 	/*

   117 	if (snprintf(th_name, sizeof (th_name) - 1, ISCSI_CONN_RXTH_NAME_FORMAT,

   106 	 * IDM CN taskq

   107 	 */

   108 

   109 	if (snprintf(th_name, sizeof (th_name) - 1,

   110 	    ISCSI_CONN_CN_TASKQ_NAME_FORMAT,

   127 	icp->conn_rx_thread = iscsi_thread_create(isp->sess_hba->hba_dip,

   120 	icp->conn_cn_taskq =

   128 	    th_name, iscsi_rx_thread, icp);

   121 	    ddi_taskq_create(icp->conn_sess->sess_hba->hba_dip, th_name, 1,

   129 

   122 	    TASKQ_DEFAULTPRI, 0);

   130 	/* Creation of the transfer thread */

   123 	if (icp->conn_cn_taskq == NULL) {

   131 	if (snprintf(th_name, sizeof (th_name) - 1, ISCSI_CONN_TXTH_NAME_FORMAT,

   132 	    icp->conn_sess->sess_hba->hba_oid, icp->conn_sess->sess_oid,

   133 	    icp->conn_oid) >= sizeof (th_name)) {

   134 		iscsi_thread_destroy(icp->conn_rx_thread);

   191 	while ((icp->conn_state != ISCSI_CONN_STATE_FREE) &&

   242 	while ((((icp->conn_state != ISCSI_CONN_STATE_FREE) &&

   192 	    (icp->conn_state != ISCSI_CONN_STATE_LOGGED_IN) &&

   243 	    (icp->conn_state != ISCSI_CONN_STATE_LOGGED_IN)) ||

   244 	    ((icp->conn_state == ISCSI_CONN_STATE_LOGGED_IN) &&

   245 	    !icp->conn_state_ffp)) &&

   205 		/* Hard case - Force connection logout */

   256 		if (icp->conn_state_ffp)

   206 		(void) iscsi_conn_state_machine(icp,

   257 			(void) iscsi_handle_logout(icp);

   207 		    ISCSI_CONN_EVENT_T9);

   258 		else {

   259 			icp->conn_state_destroy = B_FALSE;

   260 			mutex_exit(&icp->conn_state_mutex);

   261 			return (ISCSI_STATUS_INTERNAL_ERROR);

   262 		}

   320 

   375 /*

   321 /*

   376  * Process the idm notifications

   322  * iscsi_conn_state_machine - This function is used to drive the

   377  */

   323  * state machine of the iscsi connection.  It takes in a connection

   378 idm_status_t

   324  * and the associated event effecting the connection.

   379 iscsi_client_notify(idm_conn_t *ic, idm_client_notify_t icn, uintptr_t data)

   325  *

   380 {

   326  * 7.1.3  Connection State Diagram for an Initiator

   381 	iscsi_cn_task_t		*cn;

   327  *      Symbolic Names for States:

   382 	iscsi_conn_t		*icp = ic->ic_handle;

   328  *      S1: FREE        - State on instantiation, or after successful

   383 	iscsi_sess_t		*isp;

   329  *                        connection closure.

   384 

   330  *      S2: IN_LOGIN    - Waiting for login process to conclude,

   385 	/*

   331  *                        possibly involving several PDU exchanges.

   386 	 * Don't access icp if the notification is CN_CONNECT_DESTROY

   332  *      S3: LOGGED_IN   - In Full Feature Phase, waiting for all internal,

   387 	 * since icp may have already been freed.

   333  *                        iSCSI, and transport events

   388 	 *

   334  *      S4: IN_LOGOUT   - Waiting for the Logout repsonse.

   389 	 * Handle CN_FFP_ENABLED and CN_CONNECT_DESTROY immediately

   335  *      S5: FAILED      - The connection has failed.  Attempting

   390 	 */

   336  *			  to reconnect.

   391 	switch (icn) {

   337  *      S6: POLLING     - The connection reconnect attempts have

   392 	case CN_CONNECT_FAIL:

   338  *                        failed.  Continue to poll at a lower

   393 	case CN_LOGIN_FAIL:

   339  *                        frequency.

   394 		/*

   340  *

   395 		 * Wakeup any thread waiting for login stuff to happen.

   341  *      States S3, S4 constitute the Full Feature Phase

   396 		 */

   342  *              of the connection.

   397 		ASSERT(icp != NULL);

   343  *

   398 		iscsi_login_update_state(icp, LOGIN_ERROR);

   344  *      The state diagram is as follows:

   399 		return (IDM_STATUS_SUCCESS);

   345  *                 -------

   400 	case CN_READY_FOR_LOGIN:

   346  *      +-------->/ S1    \<------------------------------+

   401 		idm_conn_hold(ic); /* Released in CN_CONNECT_LOST */

   347  *      |      +->\       /<---+            /---\         |

   402 		mutex_enter(&icp->conn_state_mutex);

   348  *      |     /    ---+---     |T7/30     T7|   |         |

   403 		icp->conn_state_idm_connected = B_TRUE;

   349  *      |    +        |        |            \->------     |

   404 		cv_broadcast(&icp->conn_state_change);

   350  *      |  T8|        |T1     /      T5       / S6   \--->|

   405 		mutex_exit(&icp->conn_state_mutex);

   351  *      |    |        |      /     +----------\      /T30 |

   406 

   352  *      |    |        V     /     /            ------     |

   407 		iscsi_login_update_state(icp, LOGIN_READY);

   353  *      |    |     ------- /     /               ^        |

   408 		return (IDM_STATUS_SUCCESS);

   354  *      |    |    / S2    \     /   T5           |T7      |

   409 	case CN_CONNECT_DESTROY:

   355  *      |    |    \       /    +-------------- --+---     |

   410 		/*

   356  *      |    |     ---+---    /               / S5   \--->|

   411 		 * We released any dependecies we had on this object in

   357  *      |    |        |      /      T14/T15   \      /T30 |

   412 		 * either CN_LOGIN_FAIL or CN_CONNECT_LOST so we just need

   358  *      |    |        |T5   /  +-------------> ------     |

   413 		 * to destroy the IDM connection now.

   359  *      |    |        |    /  /                           |

   414 		 */

   360  *      |    |        |   /  /         T11                |

   415 		idm_ini_conn_destroy(ic);

   361  *      |    |        |  /  /         +----+              |

   416 		return (IDM_STATUS_SUCCESS);

   362  *      |    |        V V  /          |    |              |

   417 	}

   363  *      |    |      ------+       ----+--  |              |

   364  *      |    +-----/ S3    \T9/11/ S4    \<+              |

   365  *      +----------\       /---->\       /----------------+

   366  *                  -------       -------        T15/T17

   367  *

   368  * The state transition table is as follows:

   369  *

   370  *         +-----+---+---+------+------+---+

   371  *         |S1   |S2 |S3 |S4    |S5    |S6 |

   372  *      ---+-----+---+---+------+------+---+

   373  *       S1|T1   |T1 | - | -    | -    |   |

   374  *      ---+-----+---+---+------+------+---+

   375  *       S2|T7/30|-  |T5 | -    | -    |   |

   376  *      ---+-----+---+---+------+------+---+

   377  *       S3|T8   |-  | - |T9/11 |T14/15|   |

   378  *      ---+-----+---+---+------+------+---+

   379  *       S4|     |-  | - |T11   |T15/17|   |

   380  *      ---+-----+---+---+------+------+---+

   381  *       S5|T30  |   |T5 |      |      |T7 |

   382  *      ---+-----+---+---+------+------+---+

   383  *       S6|T30  |   |T5 |      |      |T7 |

   384  *      ---+-----+---+---+------+------+---+

   385  *

   386  * Events definitions:

   387  *

   388  * -T1: Transport connection request was made (e.g., TCP SYN sent).

   389  * -T5: The final iSCSI Login response with a Status-Class of zero was

   390  *      received.

   391  * -T7: One of the following events caused the transition:

   392  *      - Login timed out.

   393  *      - A transport disconnect indication was received.

   394  *      - A transport reset was received.

   395  *      - An internal event indicating a transport timeout was

   396  *        received.

   397  *      - An internal event of receiving a Logout repsonse (success)

   398  *        on another connection for a "close the session" Logout

   399  *        request was received.

   400  *      * In all these cases, the transport connection is closed.

   401  * -T8: An internal event of receiving a Logout response (success)

   402  *      on another connection for a "close the session" Logout request

   403  *      was received, thus closing this connection requiring no further

   404  *      cleanup.

   405  * -T9: An internal event that indicates the readiness to start the

   406  *      Logout process was received, thus prompting an iSCSI Logout to

   407  *      be sent by the initiator.

   408  * -T11: Async PDU with AsyncEvent "Request Logout" was received.

   409  * -T13: An iSCSI Logout response (success) was received, or an internal

   410  *      event of receiving a Logout response (success) on another

   411  *      connection was received.

   412  * -T14: One or more of the following events case this transition:

   413  *	- Header Digest Error

   414  *	- Protocol Error

   415  * -T15: One or more of the following events caused this transition:

   416  *      - Internal event that indicates a transport connection timeout

   417  *        was received thus prompting transport RESET or transport

   418  *        connection closure.

   419  *      - A transport RESET

   420  *      - A transport disconnect indication.

   421  *      - Async PDU with AsyncEvent "Drop connection" (for this CID)

   422  *      - Async PDU with AsyncEvent "Drop all connections"

   423  * -T17: One or more of the following events caused this transition:

   424  *      - Logout response, (failure i.e., a non-zero status) was

   425  *      received, or Logout timed out.

   426  *      - Any of the events specified for T15.

   427  * -T30: One of the following event caused the transition:

   428  *	- Thefinal iSCSI Login response was received with a non-zero

   429  *	  Status-Class.

   430  */

   431 iscsi_status_t

   432 iscsi_conn_state_machine(iscsi_conn_t *icp, iscsi_conn_event_t event)

   433 {

   434 	iscsi_status_t	    status = ISCSI_STATUS_SUCCESS;

   437 	ASSERT(mutex_owned(&icp->conn_state_mutex));

   420 	isp = icp->conn_sess;

   438 

   421 

   439 	DTRACE_PROBE3(event, iscsi_conn_t *, icp,

   422 	/*

   440 	    char *, iscsi_conn_state_str(icp->conn_state),

   423 	 * Dispatch notifications to the taskq since they often require

   441 	    char *, iscsi_conn_event_str(event));

   424 	 * long blocking operations.  In the case of CN_CONNECT_DESTROY

   442 

   425 	 * we actually just want to destroy the connection which we

   443 	icp->conn_prev_state = icp->conn_state;

   426 	 * can't do in the IDM taskq context.

   444 	icp->conn_state_lbolt = ddi_get_lbolt();

   427 	 */

   445 

   428 	cn = kmem_alloc(sizeof (*cn), KM_SLEEP);

   446 	switch (icp->conn_state) {

   429 

   447 	case ISCSI_CONN_STATE_FREE:

   430 	cn->ct_ic = ic;

   448 		status = iscsi_conn_state_free(icp, event);

   431 	cn->ct_icn = icn;

   432 	cn->ct_data = data;

   433 

   434 	idm_conn_hold(ic);

   435 

   436 	if (ddi_taskq_dispatch(icp->conn_cn_taskq,

   437 	    iscsi_client_notify_task, cn, DDI_SLEEP) != DDI_SUCCESS) {

   438 		idm_conn_rele(ic);

   439 		cmn_err(CE_WARN, "iscsi connection(%u) failure - "

   440 		    "unable to schedule notify task", icp->conn_oid);

   441 		iscsi_conn_update_state(icp, ISCSI_CONN_STATE_FREE);

   442 		mutex_enter(&isp->sess_state_mutex);

   443 		iscsi_sess_state_machine(isp,

   444 		    ISCSI_SESS_EVENT_N6);

   445 		mutex_exit(&isp->sess_state_mutex);

   446 	}

   447 

   448 	return (IDM_STATUS_SUCCESS);

   449 }

   450 

   451 static void

   452 iscsi_client_notify_task(void *cn_task_void)

   453 {

   454 	iscsi_cn_task_t		*cn_task = cn_task_void;

   455 	iscsi_conn_t		*icp;

   456 	iscsi_sess_t		*isp;

   457 	idm_conn_t		*ic;

   458 	idm_client_notify_t	icn;

   459 	uintptr_t		data;

   460 	idm_ffp_disable_t	disable_type;

   461 	boolean_t		in_login;

   462 

   463 	ic = cn_task->ct_ic;

   464 	icn = cn_task->ct_icn;

   465 	data = cn_task->ct_data;

   466 

   467 	icp = ic->ic_handle;

   468 	ASSERT(icp != NULL);

   469 	isp = icp->conn_sess;

   470 

   471 	switch (icn) {

   472 	case CN_FFP_ENABLED:

   473 		mutex_enter(&icp->conn_state_mutex);

   474 		icp->conn_async_logout = B_FALSE;

   475 		icp->conn_state_ffp = B_TRUE;

   476 		cv_broadcast(&icp->conn_state_change);

   477 		mutex_exit(&icp->conn_state_mutex);

   478 

   479 		/*

   480 		 * This logic assumes that the IDM login-snooping code

   481 		 * and the initiator login code will agree on whether

   482 		 * the connection is in FFP.  The reason we do this

   483 		 * is that we don't want to process CN_FFP_DISABLED until

   484 		 * CN_FFP_ENABLED has been full handled.

   485 		 */

   486 		mutex_enter(&icp->conn_login_mutex);

   487 		while (icp->conn_login_state != LOGIN_FFP) {

   488 			cv_wait(&icp->conn_login_cv, &icp->conn_login_mutex);

   489 		}

   490 		mutex_exit(&icp->conn_login_mutex);

   450 	case ISCSI_CONN_STATE_IN_LOGIN:

   492 	case CN_FFP_DISABLED:

   451 		iscsi_conn_state_in_login(icp, event);

   493 		disable_type = (idm_ffp_disable_t)data;

   494 

   495 		mutex_enter(&icp->conn_state_mutex);

   496 		switch (disable_type) {

   497 		case FD_SESS_LOGOUT:

   498 		case FD_CONN_LOGOUT:

   499 			if (icp->conn_async_logout) {

   500 				/*

   501 				 * Our logout was in response to an

   502 				 * async logout request so treat this

   503 				 * like a connection failure (we will

   504 				 * try to re-establish the connection)

   505 				 */

   506 				iscsi_conn_update_state_locked(icp,

   507 				    ISCSI_CONN_STATE_FAILED);

   508 			} else {

   509 				/*

   510 				 * Logout due to to user config change,

   511 				 * we will not try to re-establish

   512 				 * the connection.

   513 				 */

   514 				iscsi_conn_update_state_locked(icp,

   515 				    ISCSI_CONN_STATE_IN_LOGOUT);

   516 				/*

   517 				 * Hold off generating the ISCSI_SESS_EVENT_N3

   518 				 * event until we get the CN_CONNECT_LOST

   519 				 * notification.  This matches the pre-IDM

   520 				 * implementation better.

   521 				 */

   522 			}

   523 			break;

   524 

   525 		case FD_CONN_FAIL:

   526 		default:

   527 			iscsi_conn_update_state_locked(icp,

   528 			    ISCSI_CONN_STATE_FAILED);

   529 			break;

   530 		}

   531 

   532 		icp->conn_state_ffp = B_FALSE;

   533 		cv_broadcast(&icp->conn_state_change);

   534 		mutex_exit(&icp->conn_state_mutex);

   535 

   453 	case ISCSI_CONN_STATE_LOGGED_IN:

   537 	case CN_CONNECT_LOST:

   454 		iscsi_conn_state_logged_in(icp, event);

   538 		/*

   455 		break;

   539 		 * We only care about CN_CONNECT_LOST if we've logged in.  IDM

   456 	case ISCSI_CONN_STATE_IN_LOGOUT:

   540 		 * sends a flag as the data payload to indicate whether we

   457 		iscsi_conn_state_in_logout(icp, event);

   541 		 * were trying to login.  The CN_LOGIN_FAIL notification

   458 		break;

   542 		 * gives us what we need to know for login failures and

   459 	case ISCSI_CONN_STATE_FAILED:

   543 		 * otherwise we will need to keep a bunch of state to know

   460 		iscsi_conn_state_failed(icp, event);

   544 		 * what CN_CONNECT_LOST means to us.

   461 		break;

   545 		 */

   462 	case ISCSI_CONN_STATE_POLLING:

   546 		in_login = (boolean_t)data;

   463 		iscsi_conn_state_polling(icp, event);

   547 		if (in_login) {

   548 			mutex_enter(&icp->conn_state_mutex);

   549 

   550 			icp->conn_state_idm_connected = B_FALSE;

   551 			cv_broadcast(&icp->conn_state_change);

   552 			mutex_exit(&icp->conn_state_mutex);

   553 

   554 			/* Release connect hold from CN_READY_FOR_LOGIN */

   555 			idm_conn_rele(ic);

   556 			break;

   557 		}

   558 

   559 		/* Any remaining commands are never going to finish */

   560 		iscsi_conn_flush_active_cmds(icp);

   561 

   562 		/*

   563 		 * The connection is no longer active so cleanup any

   564 		 * references to the connection and release any holds so

   565 		 * that IDM can finish cleanup.

   566 		 */

   567 		mutex_enter(&icp->conn_state_mutex);

   568 		if (icp->conn_state != ISCSI_CONN_STATE_FAILED) {

   569 

   570 			mutex_enter(&isp->sess_state_mutex);

   571 			iscsi_sess_state_machine(isp, ISCSI_SESS_EVENT_N3);

   572 			mutex_exit(&isp->sess_state_mutex);

   573 

   574 			iscsi_conn_update_state_locked(icp,

   575 			    ISCSI_CONN_STATE_FREE);

   576 		} else {

   577 

   578 			mutex_enter(&isp->sess_state_mutex);

   579 			iscsi_sess_state_machine(isp,

   580 			    ISCSI_SESS_EVENT_N5);

   581 			mutex_exit(&isp->sess_state_mutex);

   582 

   583 			/*

   584 			 * If session type is NORMAL, try to reestablish the

   585 			 * connection.

   586 			 */

   587 			if (isp->sess_type == ISCSI_SESS_TYPE_NORMAL) {

   588 				iscsi_conn_retry(isp, icp);

   589 			} else {

   590 

   591 				mutex_enter(&isp->sess_state_mutex);

   592 				iscsi_sess_state_machine(isp,

   593 				    ISCSI_SESS_EVENT_N6);

   594 				mutex_exit(&isp->sess_state_mutex);

   595 

   596 				iscsi_conn_update_state_locked(icp,

   597 				    ISCSI_CONN_STATE_FREE);

   598 			}

   599 		}

   600 

   601 		(void) iscsi_thread_stop(icp->conn_tx_thread);

   602 

   603 		icp->conn_state_idm_connected = B_FALSE;

   604 		cv_broadcast(&icp->conn_state_change);

   605 		mutex_exit(&icp->conn_state_mutex);

   606 

   607 		/* Release connect hold from CN_READY_FOR_LOGIN */

   608 		idm_conn_rele(ic);

   466 		ASSERT(FALSE);

   611 		ISCSI_CONN_LOG(CE_WARN,

   467 		status = ISCSI_STATUS_INTERNAL_ERROR;

   612 		    "iscsi_client_notify: unknown notification: "

   468 	}

   613 		    "%x: NOT IMPLEMENTED YET: icp: %p ic: %p ",

   469 

   614 		    icn, (void *)icp, (void *)ic);

   470 	cv_broadcast(&icp->conn_state_change);

   615 		break;

   471 	return (status);

   616 	}

   472 }

   617 	/* free the task notify structure we allocated in iscsi_client_notify */

   473 

   618 	kmem_free(cn_task, sizeof (*cn_task));

   474 

   619 

   475 /*

   620 	/* Release the hold we acquired in iscsi_client_notify */

   476  * iscsi_conn_state_str - converts state enum to a string

   621 	idm_conn_rele(ic);

   477  */

   622 }

   478 char *

   479 iscsi_conn_state_str(iscsi_conn_state_t state)

   480 {

   481 	switch (state) {

   482 	case ISCSI_CONN_STATE_FREE:

   483 		return ("free");

   484 	case ISCSI_CONN_STATE_IN_LOGIN:

   485 		return ("in_login");

   486 	case ISCSI_CONN_STATE_LOGGED_IN:

   487 		return ("logged_in");

   488 	case ISCSI_CONN_STATE_IN_LOGOUT:

   489 		return ("in_logout");

   490 	case ISCSI_CONN_STATE_FAILED:

   491 		return ("failed");

   492 	case ISCSI_CONN_STATE_POLLING:

   493 		return ("polling");

   494 	default:

   495 		return ("unknown");

   496 	}

   497 }

   498 

   565 				switch (param_id) {

   689 

   690 			switch (param_id) {

  1279 }

   929 

  1280 

   930 	/* Wait for IDM abort queue to drain (if necessary) */

  1281 

   931 	mutex_enter(&icp->conn_queue_idm_aborting.mutex);

  1282 /*

   932 	while (icp->conn_queue_idm_aborting.count) {

  1283  * iscsi_conn_logged_in - connection has successfully logged in

   933 		mutex_exit(&icp->conn_queue_idm_aborting.mutex);

  1284  */

   934 		delay(drv_usectohz(100000));

  1285 static void

   935 		mutex_enter(&icp->conn_queue_idm_aborting.mutex);

  1286 iscsi_conn_logged_in(iscsi_sess_t *isp, iscsi_conn_t *icp)

   936 	}

  1287 {

   937 	mutex_exit(&icp->conn_queue_idm_aborting.mutex);

  1288 	ASSERT(isp != NULL);

  1289 	ASSERT(icp != NULL);

  1290 

  1291 	icp->conn_state = ISCSI_CONN_STATE_LOGGED_IN;

  1292 	/*

  1293 	 * We need to drop the connection state lock

  1294 	 * before updating the session state.  On update

  1295 	 * of the session state it will enumerate the

  1296 	 * target.  If we hold the lock during enumeration