provider: efa fabric: efa domain: rdmap0s6-rdm version: 119.0 type: FI_EP_RDM protocol: FI_PROTO_EFA provider: efa fabric: efa domain: rdmap0s6-dgrm version: 119.0 type: FI_EP_DGRAM protocol: FI_PROTO_EFA provider: efa fabric: efa domain: rdmap0s6-rdm version: 119.0 type: FI_EP_RDM protocol: FI_PROTO_EFA provider: efa fabric: efa domain: rdmap0s6-dgrm version: 119.0 type: FI_EP_DGRAM protocol: FI_PROTO_EFA 1: lo: mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever inet6 ::1/128 scope host valid_lft forever preferred_lft forever 2: eth0: mtu 9001 qdisc mq state UP group default qlen 1000 link/ether 06:08:67:14:35:59 brd ff:ff:ff:ff:ff:ff altname enp0s5 altname ens5 inet 10.0.16.221/20 brd 10.0.31.255 scope global dynamic noprefixroute eth0 valid_lft 3562sec preferred_lft 3562sec inet6 fe80::408:67ff:fe14:3559/64 scope link valid_lft forever preferred_lft forever 1: lo: mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever inet6 ::1/128 scope host valid_lft forever preferred_lft forever 2: eth0: mtu 9001 qdisc mq state UP group default qlen 1000 link/ether 06:1c:04:4f:9f:91 brd ff:ff:ff:ff:ff:ff altname enp0s5 altname ens5 inet 10.0.22.105/20 brd 10.0.31.255 scope global dynamic noprefixroute eth0 valid_lft 3562sec preferred_lft 3562sec inet6 fe80::41c:4ff:fe4f:9f91/64 scope link valid_lft forever preferred_lft forever c5n18-dy-c5n18xlarge-1 c5n18-dy-c5n18xlarge-2 libfabric:4078:1704470458::core:core:fi_param_get_():373 variable perf_cntr= libfabric:4078:1704470458::core:core:fi_param_get_():373 variable hook= libfabric:4088:1704470458::core:core:fi_param_get_():373 variable perf_cntr= libfabric:4088:1704470458::core:core:fi_param_get_():373 variable hook= libfabric:4078:1704470458::core:core:cuda_hmem_dl_init():431 Failed to dlopen libcudart.so libfabric:4078:1704470458::core:core:ofi_hmem_init():444 Hmem iface FI_HMEM_CUDA not supported libfabric:4078:1704470458::core:core:ofi_hmem_init():444 Hmem iface FI_HMEM_ROCR not supported libfabric:4078:1704470458::core:core:ofi_hmem_init():444 Hmem iface FI_HMEM_ZE not supported libfabric:4078:1704470458::core:core:neuron_dl_init():66 Failed to dlopen libnrt.so.1 libfabric:4078:1704470458::core:core:ofi_hmem_init():444 Hmem iface FI_HMEM_NEURON not supported libfabric:4078:1704470458::core:core:synapseai_dl_init():60 Failed to dlopen libSynapse.so libfabric:4078:1704470458::core:core:ofi_hmem_init():444 Hmem iface FI_HMEM_SYNAPSEAI not supported libfabric:4078:1704470458::core:core:fi_param_get_():373 variable hmem_disable_p2p= libfabric:4078:1704470458::core:core:fi_param_get_():373 variable mr_cache_max_size= libfabric:4078:1704470458::core:core:fi_param_get_():373 variable mr_cache_max_count= libfabric:4078:1704470458::core:core:fi_param_get_():373 variable mr_cache_monitor= libfabric:4078:1704470458::core:core:fi_param_get_():373 variable mr_cuda_cache_monitor_enabled= libfabric:4078:1704470458::core:core:fi_param_get_():373 variable mr_rocr_cache_monitor_enabled= libfabric:4078:1704470458::core:core:fi_param_get_():373 variable mr_ze_cache_monitor_enabled= libfabric:4078:1704470458::core:mr:ofi_default_cache_size():79 default cache size=1373431324 libfabric:4078:1704470458::core:core:fi_param_get_():382 read string var provider=EFA libfabric:4078:1704470458::core:core:fi_param_get_():373 variable universe_size= libfabric:4078:1704470458::core:core:fi_param_get_():373 variable av_remove_cleanup= libfabric:4078:1704470458::core:core:fi_param_get_():373 variable offload_coll_provider= libfabric:4078:1704470458::core:core:fi_param_get_():373 variable provider_path= libfabric:4078:1704470458::shm:core:fi_param_get_():373 variable sar_threshold= libfabric:4078:1704470458::shm:core:fi_param_get_():373 variable tx_size= libfabric:4078:1704470458::shm:core:fi_param_get_():373 variable rx_size= libfabric:4078:1704470458::shm:core:fi_param_get_():373 variable disable_cma= libfabric:4078:1704470458::shm:core:fi_param_get_():373 variable use_dsa_sar= libfabric:4078:1704470458::core:core:ofi_register_provider():504 registering provider: shm (119.0) libfabric:4078:1704470458::core:core:ofi_register_provider():531 "shm" filtered by provider include/exclude list, skipping libfabric:4078:1704470458::core:core:ofi_register_provider():504 registering provider: sm2 (119.0) libfabric:4078:1704470458::core:core:ofi_register_provider():531 "sm2" filtered by provider include/exclude list, skipping libfabric:4078:1704470458::ofi_rxm:core:fi_param_get_():373 variable enable_passthru= libfabric:4078:1704470458::ofi_rxm:core:fi_param_get_():373 variable buffer_size= libfabric:4078:1704470458::ofi_rxm:core:fi_param_get_():373 variable tx_size= libfabric:4078:1704470458::ofi_rxm:core:fi_param_get_():373 variable rx_size= libfabric:4078:1704470458::ofi_rxm:core:fi_param_get_():373 variable msg_tx_size= libfabric:4078:1704470458::ofi_rxm:core:fi_param_get_():373 variable msg_rx_size= libfabric:4078:1704470458::ofi_rxm:core:fi_param_get_():373 variable cm_progress_interval= libfabric:4078:1704470458::ofi_rxm:core:fi_param_get_():373 variable cq_eq_fairness= libfabric:4078:1704470458::ofi_rxm:core:fi_param_get_():373 variable data_auto_progress= libfabric:4078:1704470458::ofi_rxm:core:fi_param_get_():373 variable use_rndv_write= libfabric:4078:1704470458::ofi_rxm:core:fi_param_get_():373 variable def_wait_obj= libfabric:4078:1704470458::ofi_rxm:core:fi_param_get_():373 variable def_tcp_wait_obj= libfabric:4078:1704470458::core:core:ofi_register_provider():504 registering provider: ofi_rxm (119.0) libfabric:4078:1704470458::ofi_mrail:core:fi_param_get_():373 variable config= libfabric:4078:1704470458::ofi_mrail:core:fi_param_get_():373 variable addr= libfabric:4078:1704470458::ofi_mrail:core:fi_param_get_():373 variable addr_strc= libfabric:4078:1704470458::ofi_mrail:core:mrail_parse_env_vars():116 unable to read FI_OFI_MRAIL_ADDR env variable libfabric:4078:1704470458::core:core:ofi_register_provider():504 registering provider: ofi_mrail (119.0) libfabric:4078:1704470458::efa:core:fi_param_get_():373 variable tx_min_credits= libfabric:4078:1704470458::efa:core:fi_param_get_():373 variable tx_queue_size= libfabric:4078:1704470458::efa:core:fi_param_get_():373 variable enable_shm_transfer= libfabric:4078:1704470458::efa:core:fi_param_get_():373 variable use_zcpy_rx= libfabric:4078:1704470458::efa:core:fi_param_get_():373 variable zcpy_rx_seed= libfabric:4078:1704470458::efa:core:fi_param_get_():373 variable shm_av_size= libfabric:4078:1704470458::efa:core:fi_param_get_():373 variable recvwin_size= libfabric:4078:1704470458::efa:core:fi_param_get_():373 variable readcopy_pool_size= libfabric:4078:1704470458::efa:core:fi_param_get_():373 variable cq_size= libfabric:4078:1704470458::efa:core:fi_param_get_():373 variable max_memcpy_size= libfabric:4078:1704470458::efa:core:fi_param_get_():373 variable mr_cache_enable= libfabric:4078:1704470458::efa:core:fi_param_get_():373 variable mr_max_cached_count= libfabric:4078:1704470458::efa:core:fi_param_get_():373 variable mr_max_cached_size= libfabric:4078:1704470458::efa:core:fi_param_get_():373 variable tx_size= libfabric:4078:1704470458::efa:core:fi_param_get_():373 variable rx_size= libfabric:4078:1704470458::efa:core:fi_param_get_():373 variable rx_copy_unexp= libfabric:4078:1704470458::efa:core:fi_param_get_():373 variable rx_copy_ooo= libfabric:4078:1704470458::efa:core:fi_param_get_():373 variable max_timeout= libfabric:4078:1704470458::efa:core:fi_param_get_():373 variable timeout_interval= libfabric:4078:1704470458::efa:core:fi_param_get_():373 variable efa_cq_read_size= libfabric:4078:1704470458::efa:core:fi_param_get_():373 variable shm_cq_read_size= libfabric:4078:1704470458::efa:core:fi_param_get_():373 variable inter_read_segment_size= libfabric:4078:1704470458::efa:core:fi_param_get_():373 variable inter_max_gdrcopy_message_size= libfabric:4078:1704470458::efa:core:fi_param_get_():373 variable use_sm2= libfabric:4078:1704470458::efa:core:fi_param_get_():373 variable use_huge_page= libfabric:4078:1704470458::core:core:ofi_register_provider():504 registering provider: efa (119.0) libfabric:4078:1704470458::core:core:ofi_register_provider():504 registering provider: udp (119.0) libfabric:4078:1704470458::core:core:ofi_register_provider():531 "udp" filtered by provider include/exclude list, skipping libfabric:4078:1704470458::core:core:ofi_register_provider():504 registering provider: sockets (119.0) libfabric:4078:1704470458::core:core:ofi_register_provider():531 "sockets" filtered by provider include/exclude list, skipping libfabric:4078:1704470458::tcp:core:fi_param_get_():373 variable prov_name= libfabric:4078:1704470458::tcp:core:fi_param_get_():373 variable port_high_range= libfabric:4078:1704470458::tcp:core:fi_param_get_():373 variable port_low_range= libfabric:4078:1704470458::tcp:core:fi_param_get_():373 variable tx_size= libfabric:4078:1704470458::tcp:core:fi_param_get_():373 variable rx_size= libfabric:4078:1704470458::tcp:core:fi_param_get_():373 variable max_inject= libfabric:4078:1704470458::tcp:core:fi_param_get_():373 variable max_saved= libfabric:4078:1704470458::tcp:core:fi_param_get_():373 variable max_saved_size= libfabric:4078:1704470458::tcp:core:fi_param_get_():373 variable max_rx_size= libfabric:4078:1704470458::tcp:core:fi_param_get_():373 variable nodelay= libfabric:4078:1704470458::tcp:core:fi_param_get_():373 variable staging_sbuf_size= libfabric:4078:1704470458::tcp:core:fi_param_get_():373 variable prefetch_rbuf_size= libfabric:4078:1704470458::tcp:core:fi_param_get_():373 variable zerocopy_size= libfabric:4078:1704470458::tcp:core:fi_param_get_():373 variable trace_msg= libfabric:4078:1704470458::tcp:core:fi_param_get_():373 variable disable_auto_progress= libfabric:4078:1704470458::tcp:core:fi_param_get_():373 variable io_uring= libfabric:4078:1704470458::core:core:ofi_register_provider():504 registering provider: tcp (119.0) libfabric:4078:1704470458::core:core:ofi_register_provider():531 "tcp" filtered by provider include/exclude list, skipping libfabric:4078:1704470458::core:core:ofi_register_provider():504 registering provider: ofi_hook_perf (119.0) libfabric:4078:1704470458::core:core:ofi_register_provider():504 registering provider: ofi_hook_trace (119.0) libfabric:4078:1704470458::core:core:ofi_register_provider():504 registering provider: ofi_hook_profile (119.0) libfabric:4078:1704470458::core:core:ofi_register_provider():504 registering provider: ofi_hook_debug (119.0) libfabric:4078:1704470458::core:core:cuda_hmem_dl_init():431 Failed to dlopen libcudart.so libfabric:4078:1704470458::core:core:ofi_hmem_init():444 Hmem iface FI_HMEM_CUDA not supported libfabric:4078:1704470458::core:core:ofi_hmem_init():444 Hmem iface FI_HMEM_ROCR not supported libfabric:4078:1704470458::core:core:ofi_hmem_init():444 Hmem iface FI_HMEM_ZE not supported libfabric:4078:1704470458::core:core:neuron_dl_init():66 Failed to dlopen libnrt.so.1 libfabric:4078:1704470458::core:core:ofi_hmem_init():444 Hmem iface FI_HMEM_NEURON not supported libfabric:4078:1704470458::core:core:synapseai_dl_init():60 Failed to dlopen libSynapse.so libfabric:4078:1704470458::core:core:ofi_hmem_init():444 Hmem iface FI_HMEM_SYNAPSEAI not supported libfabric:4078:1704470458::core:core:fi_param_get_():373 variable hmem_disable_p2p= libfabric:4078:1704470458::core:core:ofi_register_provider():504 registering provider: ofi_hook_hmem (119.0) libfabric:4078:1704470458::core:core:ofi_register_provider():504 registering provider: ofi_hook_dmabuf_peer_mem (119.0) libfabric:4078:1704470458::core:core:ofi_register_provider():504 registering provider: ofi_hook_noop (119.0) libfabric:4078:1704470458::core:core:ofi_register_provider():504 registering provider: off_coll (119.0) libfabric:4078:1704470458::efa:core:ofi_check_ep_type():667 unsupported endpoint type libfabric:4078:1704470458::efa:core:ofi_check_ep_type():668 Supported: FI_EP_DGRAM libfabric:4078:1704470458::efa:core:ofi_check_ep_type():668 Requested: FI_EP_RDM libfabric:4078:1704470458::ofi_rxm:core:fi_param_get_():373 variable use_srx= libfabric:4078:1704470458:ofi_rxm:core:core:ofi_layering_ok():1168 Provider ofi_rxm is excluded libfabric:4078:1704470458:ofi_rxm:core:core:ofi_layering_ok():1179 Need core provider, skipping ofi_mrail libfabric:4078:1704470458::ofi_rxm:core:fi_param_get_():373 variable use_srx= libfabric:4078:1704470458:ofi_rxm:core:core:ofi_layering_ok():1168 Provider ofi_rxm is excluded libfabric:4078:1704470458:ofi_rxm:core:core:ofi_layering_ok():1179 Need core provider, skipping ofi_mrail libfabric:4078:1704470458::ofi_rxm:core:fi_param_get_():373 variable use_srx= libfabric:4078:1704470458:ofi_rxm:core:core:ofi_layering_ok():1185 Skipping util;efa layering libfabric:4078:1704470458:ofi_rxm:core:core:ofi_layering_ok():1179 Need core provider, skipping ofi_rxm libfabric:4078:1704470458:ofi_rxm:core:core:ofi_layering_ok():1179 Need core provider, skipping ofi_mrail libfabric:4078:1704470458::ofi_rxm:core:fi_param_get_():373 variable use_srx= libfabric:4078:1704470458:ofi_rxm:core:core:ofi_layering_ok():1185 Skipping util;efa layering libfabric:4078:1704470458:ofi_rxm:core:core:ofi_layering_ok():1179 Need core provider, skipping ofi_rxm libfabric:4078:1704470458:ofi_rxm:core:core:ofi_layering_ok():1179 Need core provider, skipping ofi_mrail libfabric:4078:1704470458::core:core:fi_getinfo_():1289 fi_getinfo: provider ofi_rxm returned -61 (No data available) libfabric:4078:1704470458::ofi_mrail:fabric:mrail_get_core_info():286 OFI_MRAIL_ADDR_STRC env variable not set! libfabric:4078:1704470458::core:core:fi_getinfo_():1289 fi_getinfo: provider ofi_mrail returned -61 (No data available) libfabric:4088:1704470458::core:core:cuda_hmem_dl_init():431 Failed to dlopen libcudart.so libfabric:4088:1704470458::core:core:ofi_hmem_init():444 Hmem iface FI_HMEM_CUDA not supported libfabric:4088:1704470458::core:core:ofi_hmem_init():444 Hmem iface FI_HMEM_ROCR not supported libfabric:4088:1704470458::core:core:ofi_hmem_init():444 Hmem iface FI_HMEM_ZE not supported libfabric:4088:1704470458::core:core:neuron_dl_init():66 Failed to dlopen libnrt.so.1 libfabric:4088:1704470458::core:core:ofi_hmem_init():444 Hmem iface FI_HMEM_NEURON not supported libfabric:4088:1704470458::core:core:synapseai_dl_init():60 Failed to dlopen libSynapse.so libfabric:4088:1704470458::core:core:ofi_hmem_init():444 Hmem iface FI_HMEM_SYNAPSEAI not supported libfabric:4088:1704470458::core:core:fi_param_get_():373 variable hmem_disable_p2p= libfabric:4088:1704470458::core:core:fi_param_get_():373 variable mr_cache_max_size= libfabric:4088:1704470458::core:core:fi_param_get_():373 variable mr_cache_max_count= libfabric:4088:1704470458::core:core:fi_param_get_():373 variable mr_cache_monitor= libfabric:4088:1704470458::core:core:fi_param_get_():373 variable mr_cuda_cache_monitor_enabled= libfabric:4088:1704470458::core:core:fi_param_get_():373 variable mr_rocr_cache_monitor_enabled= libfabric:4088:1704470458::core:core:fi_param_get_():373 variable mr_ze_cache_monitor_enabled= libfabric:4088:1704470458::core:mr:ofi_default_cache_size():79 default cache size=1373431324 libfabric:4088:1704470458::core:core:fi_param_get_():382 read string var provider=EFA libfabric:4088:1704470458::core:core:fi_param_get_():373 variable universe_size= libfabric:4088:1704470458::core:core:fi_param_get_():373 variable av_remove_cleanup= libfabric:4088:1704470458::core:core:fi_param_get_():373 variable offload_coll_provider= libfabric:4088:1704470458::core:core:fi_param_get_():373 variable provider_path= libfabric:4088:1704470458::shm:core:fi_param_get_():373 variable sar_threshold= libfabric:4088:1704470458::shm:core:fi_param_get_():373 variable tx_size= libfabric:4088:1704470458::shm:core:fi_param_get_():373 variable rx_size= libfabric:4088:1704470458::shm:core:fi_param_get_():373 variable disable_cma= libfabric:4088:1704470458::shm:core:fi_param_get_():373 variable use_dsa_sar= libfabric:4088:1704470458::core:core:ofi_register_provider():504 registering provider: shm (119.0) libfabric:4088:1704470458::core:core:ofi_register_provider():531 "shm" filtered by provider include/exclude list, skipping libfabric:4088:1704470458::core:core:ofi_register_provider():504 registering provider: sm2 (119.0) libfabric:4088:1704470458::core:core:ofi_register_provider():531 "sm2" filtered by provider include/exclude list, skipping libfabric:4088:1704470458::ofi_rxm:core:fi_param_get_():373 variable enable_passthru= libfabric:4088:1704470458::ofi_rxm:core:fi_param_get_():373 variable buffer_size= libfabric:4088:1704470458::ofi_rxm:core:fi_param_get_():373 variable tx_size= libfabric:4088:1704470458::ofi_rxm:core:fi_param_get_():373 variable rx_size= libfabric:4088:1704470458::ofi_rxm:core:fi_param_get_():373 variable msg_tx_size= libfabric:4088:1704470458::ofi_rxm:core:fi_param_get_():373 variable msg_rx_size= libfabric:4088:1704470458::ofi_rxm:core:fi_param_get_():373 variable cm_progress_interval= libfabric:4088:1704470458::ofi_rxm:core:fi_param_get_():373 variable cq_eq_fairness= libfabric:4088:1704470458::ofi_rxm:core:fi_param_get_():373 variable data_auto_progress= libfabric:4088:1704470458::ofi_rxm:core:fi_param_get_():373 variable use_rndv_write= libfabric:4088:1704470458::ofi_rxm:core:fi_param_get_():373 variable def_wait_obj= libfabric:4088:1704470458::ofi_rxm:core:fi_param_get_():373 variable def_tcp_wait_obj= libfabric:4088:1704470458::core:core:ofi_register_provider():504 registering provider: ofi_rxm (119.0) libfabric:4088:1704470458::ofi_mrail:core:fi_param_get_():373 variable config= libfabric:4088:1704470458::ofi_mrail:core:fi_param_get_():373 variable addr= libfabric:4088:1704470458::ofi_mrail:core:fi_param_get_():373 variable addr_strc= libfabric:4088:1704470458::ofi_mrail:core:mrail_parse_env_vars():116 unable to read FI_OFI_MRAIL_ADDR env variable libfabric:4088:1704470458::core:core:ofi_register_provider():504 registering provider: ofi_mrail (119.0) libfabric:4088:1704470458::efa:core:fi_param_get_():373 variable tx_min_credits= libfabric:4088:1704470458::efa:core:fi_param_get_():373 variable tx_queue_size= libfabric:4088:1704470458::efa:core:fi_param_get_():373 variable enable_shm_transfer= libfabric:4088:1704470458::efa:core:fi_param_get_():373 variable use_zcpy_rx= libfabric:4088:1704470458::efa:core:fi_param_get_():373 variable zcpy_rx_seed= libfabric:4088:1704470458::efa:core:fi_param_get_():373 variable shm_av_size= libfabric:4088:1704470458::efa:core:fi_param_get_():373 variable recvwin_size= libfabric:4088:1704470458::efa:core:fi_param_get_():373 variable readcopy_pool_size= libfabric:4088:1704470458::efa:core:fi_param_get_():373 variable cq_size= libfabric:4088:1704470458::efa:core:fi_param_get_():373 variable max_memcpy_size= libfabric:4088:1704470458::efa:core:fi_param_get_():373 variable mr_cache_enable= libfabric:4088:1704470458::efa:core:fi_param_get_():373 variable mr_max_cached_count= libfabric:4088:1704470458::efa:core:fi_param_get_():373 variable mr_max_cached_size= libfabric:4088:1704470458::efa:core:fi_param_get_():373 variable tx_size= libfabric:4088:1704470458::efa:core:fi_param_get_():373 variable rx_size= libfabric:4088:1704470458::efa:core:fi_param_get_():373 variable rx_copy_unexp= libfabric:4088:1704470458::efa:core:fi_param_get_():373 variable rx_copy_ooo= libfabric:4088:1704470458::efa:core:fi_param_get_():373 variable max_timeout= libfabric:4088:1704470458::efa:core:fi_param_get_():373 variable timeout_interval= libfabric:4088:1704470458::efa:core:fi_param_get_():373 variable efa_cq_read_size= libfabric:4088:1704470458::efa:core:fi_param_get_():373 variable shm_cq_read_size= libfabric:4088:1704470458::efa:core:fi_param_get_():373 variable inter_read_segment_size= libfabric:4088:1704470458::efa:core:fi_param_get_():373 variable inter_max_gdrcopy_message_size= libfabric:4088:1704470458::efa:core:fi_param_get_():373 variable use_sm2= libfabric:4088:1704470458::efa:core:fi_param_get_():373 variable use_huge_page= libfabric:4088:1704470458::core:core:ofi_register_provider():504 registering provider: efa (119.0) libfabric:4088:1704470458::core:core:ofi_register_provider():504 registering provider: udp (119.0) libfabric:4088:1704470458::core:core:ofi_register_provider():531 "udp" filtered by provider include/exclude list, skipping libfabric:4088:1704470458::core:core:ofi_register_provider():504 registering provider: sockets (119.0) libfabric:4088:1704470458::core:core:ofi_register_provider():531 "sockets" filtered by provider include/exclude list, skipping libfabric:4088:1704470458::tcp:core:fi_param_get_():373 variable prov_name= libfabric:4088:1704470458::tcp:core:fi_param_get_():373 variable port_high_range= libfabric:4088:1704470458::tcp:core:fi_param_get_():373 variable port_low_range= libfabric:4088:1704470458::tcp:core:fi_param_get_():373 variable tx_size= libfabric:4088:1704470458::tcp:core:fi_param_get_():373 variable rx_size= libfabric:4088:1704470458::tcp:core:fi_param_get_():373 variable max_inject= libfabric:4088:1704470458::tcp:core:fi_param_get_():373 variable max_saved= libfabric:4088:1704470458::tcp:core:fi_param_get_():373 variable max_saved_size= libfabric:4088:1704470458::tcp:core:fi_param_get_():373 variable max_rx_size= libfabric:4088:1704470458::tcp:core:fi_param_get_():373 variable nodelay= libfabric:4088:1704470458::tcp:core:fi_param_get_():373 variable staging_sbuf_size= libfabric:4088:1704470458::tcp:core:fi_param_get_():373 variable prefetch_rbuf_size= libfabric:4088:1704470458::tcp:core:fi_param_get_():373 variable zerocopy_size= libfabric:4088:1704470458::tcp:core:fi_param_get_():373 variable trace_msg= libfabric:4088:1704470458::tcp:core:fi_param_get_():373 variable disable_auto_progress= libfabric:4088:1704470458::tcp:core:fi_param_get_():373 variable io_uring= libfabric:4088:1704470458::core:core:ofi_register_provider():504 registering provider: tcp (119.0) libfabric:4088:1704470458::core:core:ofi_register_provider():531 "tcp" filtered by provider include/exclude list, skipping libfabric:4088:1704470458::core:core:ofi_register_provider():504 registering provider: ofi_hook_perf (119.0) libfabric:4088:1704470458::core:core:ofi_register_provider():504 registering provider: ofi_hook_trace (119.0) libfabric:4088:1704470458::core:core:ofi_register_provider():504 registering provider: ofi_hook_profile (119.0) libfabric:4088:1704470458::core:core:ofi_register_provider():504 registering provider: ofi_hook_debug (119.0) libfabric:4088:1704470458::core:core:cuda_hmem_dl_init():431 Failed to dlopen libcudart.so libfabric:4088:1704470458::core:core:ofi_hmem_init():444 Hmem iface FI_HMEM_CUDA not supported libfabric:4088:1704470458::core:core:ofi_hmem_init():444 Hmem iface FI_HMEM_ROCR not supported libfabric:4088:1704470458::core:core:ofi_hmem_init():444 Hmem iface FI_HMEM_ZE not supported libfabric:4088:1704470458::core:core:neuron_dl_init():66 Failed to dlopen libnrt.so.1 libfabric:4088:1704470458::core:core:ofi_hmem_init():444 Hmem iface FI_HMEM_NEURON not supported libfabric:4088:1704470458::core:core:synapseai_dl_init():60 Failed to dlopen libSynapse.so libfabric:4088:1704470458::core:core:ofi_hmem_init():444 Hmem iface FI_HMEM_SYNAPSEAI not supported libfabric:4088:1704470458::core:core:fi_param_get_():373 variable hmem_disable_p2p= libfabric:4088:1704470458::core:core:ofi_register_provider():504 registering provider: ofi_hook_hmem (119.0) libfabric:4088:1704470458::core:core:ofi_register_provider():504 registering provider: ofi_hook_dmabuf_peer_mem (119.0) libfabric:4088:1704470458::core:core:ofi_register_provider():504 registering provider: ofi_hook_noop (119.0) libfabric:4088:1704470458::core:core:ofi_register_provider():504 registering provider: off_coll (119.0) libfabric:4088:1704470458::efa:core:ofi_check_ep_type():667 unsupported endpoint type libfabric:4088:1704470458::efa:core:ofi_check_ep_type():668 Supported: FI_EP_DGRAM libfabric:4088:1704470458::efa:core:ofi_check_ep_type():668 Requested: FI_EP_RDM libfabric:4088:1704470458::ofi_rxm:core:fi_param_get_():373 variable use_srx= libfabric:4088:1704470458:ofi_rxm:core:core:ofi_layering_ok():1168 Provider ofi_rxm is excluded libfabric:4088:1704470458:ofi_rxm:core:core:ofi_layering_ok():1179 Need core provider, skipping ofi_mrail libfabric:4088:1704470458::ofi_rxm:core:fi_param_get_():373 variable use_srx= libfabric:4088:1704470458:ofi_rxm:core:core:ofi_layering_ok():1168 Provider ofi_rxm is excluded libfabric:4088:1704470458:ofi_rxm:core:core:ofi_layering_ok():1179 Need core provider, skipping ofi_mrail libfabric:4088:1704470458::ofi_rxm:core:fi_param_get_():373 variable use_srx= libfabric:4088:1704470458:ofi_rxm:core:core:ofi_layering_ok():1185 Skipping util;efa layering libfabric:4088:1704470458:ofi_rxm:core:core:ofi_layering_ok():1179 Need core provider, skipping ofi_rxm libfabric:4088:1704470458:ofi_rxm:core:core:ofi_layering_ok():1179 Need core provider, skipping ofi_mrail libfabric:4088:1704470458::ofi_rxm:core:fi_param_get_():373 variable use_srx= libfabric:4088:1704470458:ofi_rxm:core:core:ofi_layering_ok():1185 Skipping util;efa layering libfabric:4088:1704470458:ofi_rxm:core:core:ofi_layering_ok():1179 Need core provider, skipping ofi_rxm libfabric:4088:1704470458:ofi_rxm:core:core:ofi_layering_ok():1179 Need core provider, skipping ofi_mrail libfabric:4088:1704470458::core:core:fi_getinfo_():1289 fi_getinfo: provider ofi_rxm returned -61 (No data available) libfabric:4088:1704470458::ofi_mrail:fabric:mrail_get_core_info():286 OFI_MRAIL_ADDR_STRC env variable not set! libfabric:4088:1704470458::core:core:fi_getinfo_():1289 fi_getinfo: provider ofi_mrail returned -61 (No data available) libfabric:4088:1704470458::core:mr:ofi_monitor_import():847 setting imported memory monitor as default libfabric:4088:1704470458::core:core:fi_fabric_():1489 Opened fabric: efa libfabric:4088:1704470458::efa:domain:efa_domain_init_device_and_pd():121 Domain rdmap0s6-rdm selected device rdmap0s6 libfabric:4088:1704470458::efa:domain:efa_mr_cache_open():176 EFA MR cache enabled, max_cnt: 235929 max_size: 92771293593 libfabric:4088:1704470458::core:core:ofi_layering_ok():1205 Skipping shm;ofi_rxm layering libfabric:4088:1704470458::shm:core:ofi_check_info():1060 Unsupported capabilities libfabric:4088:1704470458::shm:core:ofi_check_info():1061 Supported: FI_MSG, FI_RMA, FI_TAGGED, FI_READ, FI_WRITE, FI_RECV, FI_SEND, FI_REMOTE_READ, FI_REMOTE_WRITE, FI_MULTI_RECV, FI_LOCAL_COMM, FI_RMA_EVENT, FI_SOURCE, FI_DIRECTED_RECV, FI_HMEM libfabric:4088:1704470458::shm:core:ofi_check_info():1061 Requested: FI_RMA, FI_ATOMIC, FI_READ, FI_WRITE, FI_REMOTE_READ, FI_REMOTE_WRITE, FI_MULTI_RECV libfabric:4088:1704470458::core:core:ofi_layering_ok():1205 Skipping shm;ofi_mrail layering libfabric:4088:1704470458::core:core:fi_fabric_():1489 Opened fabric: shm libfabric:4088:1704470458::shm:core:ofi_check_rx_attr():805 Tx only caps ignored in Rx caps libfabric:4088:1704470458::shm:core:ofi_check_tx_attr():903 Rx only caps ignored in Tx caps libfabric:4088:1704470458::shm:core:ofi_check_info():1060 Unsupported capabilities libfabric:4088:1704470458::shm:core:ofi_check_info():1061 Supported: FI_MSG, FI_RMA, FI_TAGGED, FI_READ, FI_WRITE, FI_RECV, FI_SEND, FI_REMOTE_READ, FI_REMOTE_WRITE, FI_MULTI_RECV, FI_LOCAL_COMM, FI_RMA_EVENT, FI_SOURCE, FI_DIRECTED_RECV, FI_HMEM libfabric:4088:1704470458::shm:core:ofi_check_info():1061 Requested: FI_RMA, FI_ATOMIC, FI_READ, FI_WRITE, FI_REMOTE_READ, FI_REMOTE_WRITE, FI_MULTI_RECV, FI_RMA_EVENT libfabric:4088:1704470458::efa:core:fi_param_get_():373 variable runt_size= libfabric:4088:1704470458::efa:core:fi_param_get_():373 variable inter_max_medium_message_size= libfabric:4088:1704470458::efa:core:fi_param_get_():373 variable inter_min_read_message_size= libfabric:4088:1704470458::efa:core:fi_param_get_():373 variable inter_min_read_write_size= libfabric:4088:1704470458::efa:domain:efa_domain_hmem_info_init_cuda():153 FI_HMEM_CUDA is not initialized libfabric:4088:1704470458::efa:domain:efa_domain_hmem_info_init_neuron():230 FI_HMEM_NEURON is not initialized libfabric:4088:1704470458::efa:domain:efa_domain_hmem_info_init_synapseai():316 FI_HMEM_SYNAPSEAI is not initialized libfabric:4088:1704470458::core:core:fi_param_get_():373 variable universe_size= libfabric:4088:1704470458::efa:av:util_av_init():486 AV size 16384 libfabric:4088:1704470458::shm:av:util_av_init():486 AV size 256 libfabric:4088:1704470458::efa:av:efa_av_open():983 fi_av_attr:0 libfabric:4088:1704470458::efa:core:ofi_check_rx_attr():805 Tx only caps ignored in Rx caps libfabric:4088:1704470458::efa:core:ofi_check_tx_attr():903 Rx only caps ignored in Tx caps libfabric:4088:1704470458::efa:core:ofi_check_ep_type():667 unsupported endpoint type libfabric:4088:1704470458::efa:core:ofi_check_ep_type():668 Supported: FI_EP_DGRAM libfabric:4088:1704470458::efa:core:ofi_check_ep_type():668 Requested: FI_EP_RDM libfabric:4088:1704470458::shm:core:ofi_check_rx_attr():805 Tx only caps ignored in Rx caps libfabric:4088:1704470458::shm:core:ofi_check_tx_attr():903 Rx only caps ignored in Tx caps libfabric:4088:1704470458::shm:core:ofi_check_info():1060 Unsupported capabilities libfabric:4088:1704470458::shm:core:ofi_check_info():1061 Supported: FI_MSG, FI_RMA, FI_TAGGED, FI_READ, FI_WRITE, FI_RECV, FI_SEND, FI_REMOTE_READ, FI_REMOTE_WRITE, FI_MULTI_RECV, FI_LOCAL_COMM, FI_RMA_EVENT, FI_SOURCE, FI_DIRECTED_RECV, FI_HMEM libfabric:4088:1704470458::shm:core:ofi_check_info():1061 Requested: FI_RMA, FI_ATOMIC, FI_READ, FI_WRITE, FI_REMOTE_READ, FI_REMOTE_WRITE, FI_MULTI_RECV, FI_RMA_EVENT libfabric:4088:1704470458::efa:ep_ctrl:efa_rdm_ep_open():452 efa_rdm_ep->host_id: i-0ef5266ddb677e20a libfabric:4088:1704470458::efa:core:fi_param_get_():373 variable use_device_rdma= libfabric:4088:1704470458::efa:cq:efa_rdm_ep_open():469 FI_DELIVERY_COMPLETE unsupported libfabric:4088:1704470458::efa:ep_ctrl:efa_rdm_ep_set_use_zcpy_rx():400 efa_rdm_ep->use_zcpy_rx = 0 libfabric:4088:1704470458::efa:ep_ctrl:efa_rdm_ep_open():490 efa_rdm_ep->handle_resource_management = 2 libfabric:4088:1704470458::efa:ep_ctrl:efa_base_ep_enable_qp():230 QP enabled! qp_n: 0 qkey: 1211585764 libfabric:4088:1704470458::efa:ep_ctrl:efa_rdm_ep_ctrl():1056 libfabric 1.19.0amzn4.0 efa endpoint created! address: fi_addr_efa://[fe80::41c:4ff:fe4f:9f91]:0:1211585764 libfabric:4088:1704470458::efa:ep_ctrl:efa_base_ep_getname():324 EP addr: GID[fe80::41c:4ff:fe4f:9f91] QP[0] QKEY[1211585764] (length 64) libfabric:4078:1704470458::core:mr:ofi_monitor_import():847 setting imported memory monitor as default libfabric:4078:1704470458::core:core:fi_fabric_():1489 Opened fabric: efa libfabric:4078:1704470458::efa:domain:efa_domain_init_device_and_pd():121 Domain rdmap0s6-rdm selected device rdmap0s6 libfabric:4078:1704470458::efa:domain:efa_mr_cache_open():176 EFA MR cache enabled, max_cnt: 235929 max_size: 92771293593 libfabric:4078:1704470458::core:core:ofi_layering_ok():1205 Skipping shm;ofi_rxm layering libfabric:4078:1704470458::shm:core:ofi_check_info():1060 Unsupported capabilities libfabric:4078:1704470458::shm:core:ofi_check_info():1061 Supported: FI_MSG, FI_RMA, FI_TAGGED, FI_READ, FI_WRITE, FI_RECV, FI_SEND, FI_REMOTE_READ, FI_REMOTE_WRITE, FI_MULTI_RECV, FI_LOCAL_COMM, FI_RMA_EVENT, FI_SOURCE, FI_DIRECTED_RECV, FI_HMEM libfabric:4078:1704470458::shm:core:ofi_check_info():1061 Requested: FI_RMA, FI_ATOMIC, FI_READ, FI_WRITE, FI_REMOTE_READ, FI_REMOTE_WRITE, FI_MULTI_RECV libfabric:4078:1704470458::core:core:ofi_layering_ok():1205 Skipping shm;ofi_mrail layering libfabric:4078:1704470458::core:core:fi_fabric_():1489 Opened fabric: shm libfabric:4078:1704470458::shm:core:ofi_check_rx_attr():805 Tx only caps ignored in Rx caps libfabric:4078:1704470458::shm:core:ofi_check_tx_attr():903 Rx only caps ignored in Tx caps libfabric:4078:1704470458::shm:core:ofi_check_info():1060 Unsupported capabilities libfabric:4078:1704470458::shm:core:ofi_check_info():1061 Supported: FI_MSG, FI_RMA, FI_TAGGED, FI_READ, FI_WRITE, FI_RECV, FI_SEND, FI_REMOTE_READ, FI_REMOTE_WRITE, FI_MULTI_RECV, FI_LOCAL_COMM, FI_RMA_EVENT, FI_SOURCE, FI_DIRECTED_RECV, FI_HMEM libfabric:4078:1704470458::shm:core:ofi_check_info():1061 Requested: FI_RMA, FI_ATOMIC, FI_READ, FI_WRITE, FI_REMOTE_READ, FI_REMOTE_WRITE, FI_MULTI_RECV, FI_RMA_EVENT libfabric:4078:1704470458::efa:core:fi_param_get_():373 variable runt_size= libfabric:4078:1704470458::efa:core:fi_param_get_():373 variable inter_max_medium_message_size= libfabric:4078:1704470458::efa:core:fi_param_get_():373 variable inter_min_read_message_size= libfabric:4078:1704470458::efa:core:fi_param_get_():373 variable inter_min_read_write_size= libfabric:4078:1704470458::efa:domain:efa_domain_hmem_info_init_cuda():153 FI_HMEM_CUDA is not initialized libfabric:4078:1704470458::efa:domain:efa_domain_hmem_info_init_neuron():230 FI_HMEM_NEURON is not initialized libfabric:4078:1704470458::efa:domain:efa_domain_hmem_info_init_synapseai():316 FI_HMEM_SYNAPSEAI is not initialized libfabric:4078:1704470458::core:core:fi_param_get_():373 variable universe_size= libfabric:4078:1704470458::efa:av:util_av_init():486 AV size 16384 libfabric:4078:1704470458::shm:av:util_av_init():486 AV size 256 libfabric:4078:1704470458::efa:av:efa_av_open():983 fi_av_attr:0 libfabric:4078:1704470458::efa:core:ofi_check_rx_attr():805 Tx only caps ignored in Rx caps libfabric:4078:1704470458::efa:core:ofi_check_tx_attr():903 Rx only caps ignored in Tx caps libfabric:4078:1704470458::efa:core:ofi_check_ep_type():667 unsupported endpoint type libfabric:4078:1704470458::efa:core:ofi_check_ep_type():668 Supported: FI_EP_DGRAM libfabric:4078:1704470458::efa:core:ofi_check_ep_type():668 Requested: FI_EP_RDM libfabric:4078:1704470458::shm:core:ofi_check_rx_attr():805 Tx only caps ignored in Rx caps libfabric:4078:1704470458::shm:core:ofi_check_tx_attr():903 Rx only caps ignored in Tx caps libfabric:4078:1704470458::shm:core:ofi_check_info():1060 Unsupported capabilities libfabric:4078:1704470458::shm:core:ofi_check_info():1061 Supported: FI_MSG, FI_RMA, FI_TAGGED, FI_READ, FI_WRITE, FI_RECV, FI_SEND, FI_REMOTE_READ, FI_REMOTE_WRITE, FI_MULTI_RECV, FI_LOCAL_COMM, FI_RMA_EVENT, FI_SOURCE, FI_DIRECTED_RECV, FI_HMEM libfabric:4078:1704470458::shm:core:ofi_check_info():1061 Requested: FI_RMA, FI_ATOMIC, FI_READ, FI_WRITE, FI_REMOTE_READ, FI_REMOTE_WRITE, FI_MULTI_RECV, FI_RMA_EVENT libfabric:4078:1704470458::efa:ep_ctrl:efa_rdm_ep_open():452 efa_rdm_ep->host_id: i-0339cefba46aa889a libfabric:4078:1704470458::efa:core:fi_param_get_():373 variable use_device_rdma= libfabric:4078:1704470458::efa:cq:efa_rdm_ep_open():469 FI_DELIVERY_COMPLETE unsupported libfabric:4078:1704470458::efa:ep_ctrl:efa_rdm_ep_set_use_zcpy_rx():400 efa_rdm_ep->use_zcpy_rx = 0 libfabric:4078:1704470458::efa:ep_ctrl:efa_rdm_ep_open():490 efa_rdm_ep->handle_resource_management = 2 libfabric:4078:1704470458::efa:ep_ctrl:efa_base_ep_enable_qp():230 QP enabled! qp_n: 0 qkey: 1842533917 libfabric:4078:1704470458::efa:ep_ctrl:efa_rdm_ep_ctrl():1056 libfabric 1.19.0amzn4.0 efa endpoint created! address: fi_addr_efa://[fe80::408:67ff:fe14:3559]:0:1842533917 libfabric:4078:1704470458::efa:ep_ctrl:efa_base_ep_getname():324 EP addr: GID[fe80::408:67ff:fe14:3559] QP[0] QKEY[1842533917] (length 64) libfabric:4088:1704470458::efa:core:ofi_check_ep_type():667 unsupported endpoint type libfabric:4088:1704470458::efa:core:ofi_check_ep_type():668 Supported: FI_EP_DGRAM libfabric:4088:1704470458::efa:core:ofi_check_ep_type():668 Requested: FI_EP_RDM libfabric:4088:1704470458::core:core:ofi_layering_ok():1205 Skipping efa;ofi_rxm layering libfabric:4088:1704470458::core:core:ofi_layering_ok():1205 Skipping efa;ofi_mrail layering libfabric:4088:1704470458::core:core:fi_fabric_():1489 Opened fabric: efa libfabric:4088:1704470458::efa:domain:efa_domain_init_device_and_pd():121 Domain rdmap0s6-rdm selected device rdmap0s6 libfabric:4088:1704470458::efa:domain:efa_mr_cache_open():176 EFA MR cache enabled, max_cnt: 235929 max_size: 92771293593 libfabric:4088:1704470458::core:core:ofi_layering_ok():1205 Skipping shm;ofi_rxm layering libfabric:4088:1704470458::core:core:ofi_layering_ok():1205 Skipping shm;ofi_mrail layering libfabric:4088:1704470458::core:core:fi_fabric_():1489 Opened fabric: shm libfabric:4088:1704470458::shm:core:ofi_check_rx_attr():805 Tx only caps ignored in Rx caps libfabric:4088:1704470458::shm:core:ofi_check_tx_attr():903 Rx only caps ignored in Tx caps libfabric:4088:1704470458::shm:core:ofi_check_rx_attr():805 Tx only caps ignored in Rx caps libfabric:4088:1704470458::shm:core:ofi_check_tx_attr():903 Rx only caps ignored in Tx caps libfabric:4088:1704470458::shm:core:ofi_check_tx_attr():940 inject_size too large libfabric:4088:1704470458::shm:core:ofi_check_tx_attr():941 Supported: 0 libfabric:4088:1704470458::shm:core:ofi_check_tx_attr():941 Requested: 4096 libfabric:4088:1704470458::efa:core:fi_param_get_():373 variable runt_size= libfabric:4088:1704470458::efa:core:fi_param_get_():373 variable inter_max_medium_message_size= libfabric:4088:1704470458::efa:core:fi_param_get_():373 variable inter_min_read_message_size= libfabric:4088:1704470458::efa:core:fi_param_get_():373 variable inter_min_read_write_size= libfabric:4088:1704470458::efa:domain:efa_domain_hmem_info_init_cuda():153 FI_HMEM_CUDA is not initialized libfabric:4088:1704470458::efa:domain:efa_domain_hmem_info_init_neuron():230 FI_HMEM_NEURON is not initialized libfabric:4088:1704470458::efa:domain:efa_domain_hmem_info_init_synapseai():316 FI_HMEM_SYNAPSEAI is not initialized libfabric:4088:1704470458::efa:core:ofi_check_rx_attr():805 Tx only caps ignored in Rx caps libfabric:4088:1704470458::efa:core:ofi_check_tx_attr():903 Rx only caps ignored in Tx caps libfabric:4088:1704470458::efa:core:ofi_check_ep_type():667 unsupported endpoint type libfabric:4088:1704470458::efa:core:ofi_check_ep_type():668 Supported: FI_EP_DGRAM libfabric:4088:1704470458::efa:core:ofi_check_ep_type():668 Requested: FI_EP_RDM libfabric:4088:1704470458::shm:core:ofi_check_rx_attr():805 Tx only caps ignored in Rx caps libfabric:4088:1704470458::shm:core:ofi_check_tx_attr():903 Rx only caps ignored in Tx caps libfabric:4088:1704470458::shm:core:ofi_check_rx_attr():805 Tx only caps ignored in Rx caps libfabric:4088:1704470458::shm:core:ofi_check_tx_attr():903 Rx only caps ignored in Tx caps libfabric:4088:1704470458::shm:core:ofi_check_tx_attr():940 inject_size too large libfabric:4088:1704470458::shm:core:ofi_check_tx_attr():941 Supported: 0 libfabric:4088:1704470458::shm:core:ofi_check_tx_attr():941 Requested: 4096 libfabric:4088:1704470458::efa:ep_ctrl:efa_rdm_ep_open():452 efa_rdm_ep->host_id: i-0ef5266ddb677e20a libfabric:4088:1704470458::efa:core:fi_param_get_():373 variable use_device_rdma= libfabric:4088:1704470458::efa:ep_ctrl:efa_rdm_ep_set_use_zcpy_rx():400 efa_rdm_ep->use_zcpy_rx = 0 libfabric:4088:1704470458::efa:ep_ctrl:efa_rdm_ep_open():490 efa_rdm_ep->handle_resource_management = 2 libfabric:4088:1704470458::core:core:fi_param_get_():373 variable universe_size= libfabric:4088:1704470458::efa:av:util_av_init():486 AV size 16384 libfabric:4088:1704470458::shm:av:util_av_init():486 AV size 256 libfabric:4088:1704470458::efa:av:efa_av_open():983 fi_av_attr:0 libfabric:4088:1704470458::efa:ep_ctrl:efa_base_ep_enable_qp():230 QP enabled! qp_n: 1 qkey: 1597341547 libfabric:4078:1704470458::efa:core:ofi_check_ep_type():667 unsupported endpoint type libfabric:4078:1704470458::efa:core:ofi_check_ep_type():668 Supported: FI_EP_DGRAM libfabric:4078:1704470458::efa:core:ofi_check_ep_type():668 Requested: FI_EP_RDM libfabric:4078:1704470458::core:core:ofi_layering_ok():1205 Skipping efa;ofi_rxm layering libfabric:4078:1704470458::core:core:ofi_layering_ok():1205 Skipping efa;ofi_mrail layering libfabric:4078:1704470458::core:core:fi_fabric_():1489 Opened fabric: efa libfabric:4078:1704470458::efa:domain:efa_domain_init_device_and_pd():121 Domain rdmap0s6-rdm selected device rdmap0s6 libfabric:4078:1704470458::efa:domain:efa_mr_cache_open():176 EFA MR cache enabled, max_cnt: 235929 max_size: 92771293593 libfabric:4078:1704470458::core:core:ofi_layering_ok():1205 Skipping shm;ofi_rxm layering libfabric:4078:1704470458::core:core:ofi_layering_ok():1205 Skipping shm;ofi_mrail layering libfabric:4088:1704470458::efa:ep_ctrl:efa_rdm_ep_ctrl():1056 libfabric 1.19.0amzn4.0 efa endpoint created! address: fi_addr_efa://[fe80::41c:4ff:fe4f:9f91]:1:1597341547 libfabric:4088:1704470458::efa:ep_ctrl:efa_base_ep_getname():324 EP addr: GID[fe80::41c:4ff:fe4f:9f91] QP[1] QKEY[1597341547] (length 64) libfabric:4078:1704470458::core:core:fi_fabric_():1489 Opened fabric: shm libfabric:4078:1704470458::shm:core:ofi_check_rx_attr():805 Tx only caps ignored in Rx caps libfabric:4078:1704470458::shm:core:ofi_check_tx_attr():903 Rx only caps ignored in Tx caps libfabric:4078:1704470458::shm:core:ofi_check_rx_attr():805 Tx only caps ignored in Rx caps libfabric:4078:1704470458::shm:core:ofi_check_tx_attr():903 Rx only caps ignored in Tx caps libfabric:4078:1704470458::shm:core:ofi_check_tx_attr():940 inject_size too large libfabric:4078:1704470458::shm:core:ofi_check_tx_attr():941 Supported: 0 libfabric:4078:1704470458::shm:core:ofi_check_tx_attr():941 Requested: 4096 libfabric:4078:1704470458::efa:core:fi_param_get_():373 variable runt_size= libfabric:4078:1704470458::efa:core:fi_param_get_():373 variable inter_max_medium_message_size= libfabric:4078:1704470458::efa:core:fi_param_get_():373 variable inter_min_read_message_size= libfabric:4078:1704470458::efa:core:fi_param_get_():373 variable inter_min_read_write_size= libfabric:4078:1704470458::efa:domain:efa_domain_hmem_info_init_cuda():153 FI_HMEM_CUDA is not initialized libfabric:4078:1704470458::efa:domain:efa_domain_hmem_info_init_neuron():230 FI_HMEM_NEURON is not initialized libfabric:4078:1704470458::efa:domain:efa_domain_hmem_info_init_synapseai():316 FI_HMEM_SYNAPSEAI is not initialized libfabric:4078:1704470458::efa:core:ofi_check_rx_attr():805 Tx only caps ignored in Rx caps libfabric:4078:1704470458::efa:core:ofi_check_tx_attr():903 Rx only caps ignored in Tx caps libfabric:4078:1704470458::efa:core:ofi_check_ep_type():667 unsupported endpoint type libfabric:4078:1704470458::efa:core:ofi_check_ep_type():668 Supported: FI_EP_DGRAM libfabric:4078:1704470458::efa:core:ofi_check_ep_type():668 Requested: FI_EP_RDM libfabric:4078:1704470458::shm:core:ofi_check_rx_attr():805 Tx only caps ignored in Rx caps libfabric:4078:1704470458::shm:core:ofi_check_tx_attr():903 Rx only caps ignored in Tx caps libfabric:4078:1704470458::shm:core:ofi_check_rx_attr():805 Tx only caps ignored in Rx caps libfabric:4078:1704470458::shm:core:ofi_check_tx_attr():903 Rx only caps ignored in Tx caps libfabric:4078:1704470458::shm:core:ofi_check_tx_attr():940 inject_size too large libfabric:4078:1704470458::shm:core:ofi_check_tx_attr():941 Supported: 0 libfabric:4078:1704470458::shm:core:ofi_check_tx_attr():941 Requested: 4096 libfabric:4078:1704470458::efa:ep_ctrl:efa_rdm_ep_open():452 efa_rdm_ep->host_id: i-0339cefba46aa889a libfabric:4078:1704470458::efa:core:fi_param_get_():373 variable use_device_rdma= libfabric:4078:1704470458::efa:ep_ctrl:efa_rdm_ep_set_use_zcpy_rx():400 efa_rdm_ep->use_zcpy_rx = 0 libfabric:4078:1704470458::efa:ep_ctrl:efa_rdm_ep_open():490 efa_rdm_ep->handle_resource_management = 2 libfabric:4078:1704470458::core:core:fi_param_get_():373 variable universe_size= libfabric:4078:1704470458::efa:av:util_av_init():486 AV size 16384 libfabric:4078:1704470458::shm:av:util_av_init():486 AV size 256 libfabric:4078:1704470458::efa:av:efa_av_open():983 fi_av_attr:0 libfabric:4078:1704470458::efa:ep_ctrl:efa_base_ep_enable_qp():230 QP enabled! qp_n: 1 qkey: 1423542940 libfabric:4078:1704470458::efa:ep_ctrl:efa_rdm_ep_ctrl():1056 libfabric 1.19.0amzn4.0 efa endpoint created! address: fi_addr_efa://[fe80::408:67ff:fe14:3559]:1:1423542940 libfabric:4078:1704470458::efa:ep_ctrl:efa_base_ep_getname():324 EP addr: GID[fe80::408:67ff:fe14:3559] QP[1] QKEY[1423542940] (length 64) libfabric:4088:1704470458::efa:av:efa_av_insert_one():638 Inserting address GID[fe80::41c:4ff:fe4f:9f91] QP[1] QKEY[1597341547] to AV .... libfabric:4088:1704470458::efa:av:ofi_av_insert_addr():291 inserting addr: (null) libfabric:4088:1704470458::efa:av:ofi_av_insert_addr():314 fi_addr: 0 libfabric:4088:1704470458::efa:av:efa_is_local_peer():64 The peer is local. libfabric:4088:1704470458::shm:av:smr_av_insert():84 fe80::41c:4ff:fe4f:9f91_0001_5f357f6b_03e8 libfabric:4088:1704470458::shm:av:ofi_av_insert_addr():291 inserting addr: (null) libfabric:4088:1704470458::shm:av:ofi_av_insert_addr():314 fi_addr: 0 libfabric:4088:1704470458::shm:av:smr_av_insert():103 fi_addr: 0 libfabric:4088:1704470458::efa:av:efa_conn_rdm_init():356 Successfully inserted fe80::41c:4ff:fe4f:9f91_0001_5f357f6b_03e8 to shm provider's av. efa_fiaddr: 0 shm_fiaddr = 0 libfabric:4088:1704470458::efa:av:efa_av_insert_one():661 Successfully inserted address GID[fe80::41c:4ff:fe4f:9f91] QP[1] QKEY[1597341547] to AV. fi_addr: 0 libfabric:4078:1704470458::efa:av:efa_av_insert_one():638 Inserting address GID[fe80::408:67ff:fe14:3559] QP[1] QKEY[1423542940] to AV .... libfabric:4078:1704470458::efa:av:ofi_av_insert_addr():291 inserting addr: (null) libfabric:4078:1704470458::efa:av:ofi_av_insert_addr():314 fi_addr: 0 libfabric:4078:1704470458::efa:av:efa_is_local_peer():64 The peer is local. libfabric:4078:1704470458::shm:av:smr_av_insert():84 fe80::408:67ff:fe14:3559_0001_54d98a9c_03e8 libfabric:4078:1704470458::shm:av:ofi_av_insert_addr():291 inserting addr: (null) libfabric:4078:1704470458::shm:av:ofi_av_insert_addr():314 fi_addr: 0 libfabric:4078:1704470458::shm:av:smr_av_insert():103 fi_addr: 0 libfabric:4078:1704470458::efa:av:efa_conn_rdm_init():356 Successfully inserted fe80::408:67ff:fe14:3559_0001_54d98a9c_03e8 to shm provider's av. efa_fiaddr: 0 shm_fiaddr = 0 libfabric:4078:1704470458::efa:av:efa_av_insert_one():661 Successfully inserted address GID[fe80::408:67ff:fe14:3559] QP[1] QKEY[1423542940] to AV. fi_addr: 0 libfabric:4088:1704470458::efa:av:efa_av_insert_one():638 Inserting address GID[fe80::408:67ff:fe14:3559] QP[1] QKEY[1423542940] to AV .... libfabric:4088:1704470458::efa:av:ofi_av_insert_addr():291 inserting addr: (null) libfabric:4088:1704470458::efa:av:ofi_av_insert_addr():314 fi_addr: 1 libfabric:4078:1704470458::efa:av:efa_av_insert_one():638 Inserting address GID[fe80::41c:4ff:fe4f:9f91] QP[1] QKEY[1597341547] to AV .... libfabric:4078:1704470458::efa:av:ofi_av_insert_addr():291 inserting addr: (null) libfabric:4078:1704470458::efa:av:ofi_av_insert_addr():314 fi_addr: 1 libfabric:4088:1704470458::efa:av:efa_av_insert_one():661 Successfully inserted address GID[fe80::408:67ff:fe14:3559] QP[1] QKEY[1423542940] to AV. fi_addr: 1 libfabric:4088:1704470458::efa:cq:efa_rdm_pke_handle_handshake_recv():108 Received peer host id: i-0339cefba46aa889a libfabric:4078:1704470458::efa:av:efa_av_insert_one():661 Successfully inserted address GID[fe80::41c:4ff:fe4f:9f91] QP[1] QKEY[1597341547] to AV. fi_addr: 1 libfabric:4078:1704470458::efa:cq:efa_rdm_pke_handle_handshake_recv():108 Received peer host id: i-0ef5266ddb677e20a libfabric:4078:1704470458::efa:cq:efa_rdm_pke_handle_handshake_recv():112 Received peer EFA device version: 0xefa0 libfabric:4088:1704470458::efa:cq:efa_rdm_pke_handle_handshake_recv():112 Received peer EFA device version: 0xefa0 #---------------------------------------------------------------- # Intel(R) MPI Benchmarks 2018, MPI-1 part #---------------------------------------------------------------- # Date : Fri Jan 5 16:00:58 2024 # Machine : x86_64 # System : Linux # Release : 4.18.0-477.10.1.el8_8.x86_64 # Version : #1 SMP Tue May 16 11:38:37 UTC 2023 # MPI Version : 3.1 # MPI Thread Environment: # Calling sequence was: # /home/rocky/mpi-benchmarks/src_c/IMB-MPI1 -msglog 3:28 PingPong # Minimum message length in bytes: 0 # Maximum message length in bytes: 268435456 # # MPI_Datatype : MPI_BYTE # MPI_Datatype for reductions : MPI_FLOAT # MPI_Op : MPI_SUM # # # List of Benchmarks to run: # PingPong #--------------------------------------------------- # Benchmarking PingPong # #processes = 2 #--------------------------------------------------- #bytes #repetitions t[usec] Mbytes/sec 0 1000 18.39 0.00 8 1000 18.89 0.42 16 1000 18.85 0.85 32 1000 18.91 1.69 64 1000 18.88 3.39 128 1000 18.93 6.76 256 1000 19.06 13.43 512 1000 19.13 26.76 1024 1000 19.52 52.45 2048 1000 20.57 99.54 4096 1000 22.74 180.12 8192 1000 27.61 296.72 16384 1000 29.48 555.70 32768 1000 34.24 956.97 65536 640 41.64 1573.99 131072 320 97.83 1339.77 262144 160 118.46 2212.87 524288 80 209.71 2500.08 1048576 40 394.70 2656.67 2097152 20 762.33 2750.97 4194304 10 1449.04 2894.53 8388608 5 2859.11 2934.00 16777216 2 5918.97 2834.48 33554432 1 12526.96 2678.58 67108864 1 25970.30 2584.06 134217728 1 50989.61 2632.26 268435456 1 102811.02 2610.96 libfabric:4088:1704470462::shm:av:smr_av_remove():165 0 libfabric:4078:1704470462::shm:av:smr_av_remove():165 0 libfabric:4088:1704470462::efa:av:efa_conn_release():597 efa_conn released! conn[0x7fa32b62e9f0] GID[fe80::41c:4ff:fe4f:9f91] QP[1] libfabric:4078:1704470462::efa:av:efa_conn_release():597 efa_conn released! conn[0x7f17cc09d9f0] GID[fe80::408:67ff:fe14:3559] QP[1] libfabric:4078:1704470462::efa:av:efa_conn_release():597 efa_conn released! conn[0x7f17cc09e370] GID[fe80::41c:4ff:fe4f:9f91] QP[1] libfabric:4088:1704470462::efa:av:efa_conn_release():597 efa_conn released! conn[0x7fa32b62f370] GID[fe80::408:67ff:fe14:3559] QP[1] libfabric:4088:1704470462::efa:mr:ofi_mr_cache_cleanup():471 MR cache stats: searches 6904, deletes 6904, hits 6887 notify 48 libfabric:4078:1704470462::efa:mr:ofi_mr_cache_cleanup():471 MR cache stats: searches 6904, deletes 6904, hits 6887 notify 48 libfabric:4088:1704470462::efa:mr:ofi_mr_cache_cleanup():471 MR cache stats: searches 0, deletes 0, hits 0 notify 61 libfabric:4078:1704470462::efa:mr:ofi_mr_cache_cleanup():471 MR cache stats: searches 0, deletes 0, hits 0 notify 61 # All processes entering MPI_Finalize c5n18-dy-c5n18xlarge-1 c5n18-dy-c5n18xlarge-2 libfabric:4137:1704470463::core:core:fi_param_get_():373 variable perf_cntr= libfabric:4137:1704470463::core:core:fi_param_get_():373 variable hook= libfabric:4137:1704470463::core:core:cuda_hmem_dl_init():431 Failed to dlopen libcudart.so libfabric:4137:1704470463::core:core:ofi_hmem_init():444 Hmem iface FI_HMEM_CUDA not supported libfabric:4137:1704470463::core:core:ofi_hmem_init():444 Hmem iface FI_HMEM_ROCR not supported libfabric:4137:1704470463::core:core:ofi_hmem_init():444 Hmem iface FI_HMEM_ZE not supported libfabric:4137:1704470463::core:core:neuron_dl_init():66 Failed to dlopen libnrt.so.1 libfabric:4137:1704470463::core:core:ofi_hmem_init():444 Hmem iface FI_HMEM_NEURON not supported libfabric:4137:1704470463::core:core:synapseai_dl_init():60 Failed to dlopen libSynapse.so libfabric:4137:1704470463::core:core:ofi_hmem_init():444 Hmem iface FI_HMEM_SYNAPSEAI not supported libfabric:4137:1704470463::core:core:fi_param_get_():373 variable hmem_disable_p2p= libfabric:4137:1704470463::core:core:fi_param_get_():373 variable mr_cache_max_size= libfabric:4137:1704470463::core:core:fi_param_get_():373 variable mr_cache_max_count= libfabric:4137:1704470463::core:core:fi_param_get_():373 variable mr_cache_monitor= libfabric:4137:1704470463::core:core:fi_param_get_():373 variable mr_cuda_cache_monitor_enabled= libfabric:4137:1704470463::core:core:fi_param_get_():373 variable mr_rocr_cache_monitor_enabled= libfabric:4137:1704470463::core:core:fi_param_get_():373 variable mr_ze_cache_monitor_enabled= libfabric:4137:1704470463::core:mr:ofi_default_cache_size():79 default cache size=1373431324 libfabric:4137:1704470463::core:core:fi_param_get_():382 read string var provider=EFA libfabric:4137:1704470463::core:core:fi_param_get_():373 variable universe_size= libfabric:4137:1704470463::core:core:fi_param_get_():373 variable av_remove_cleanup= libfabric:4137:1704470463::core:core:fi_param_get_():373 variable offload_coll_provider= libfabric:4137:1704470463::core:core:fi_param_get_():373 variable provider_path= libfabric:4137:1704470463::shm:core:fi_param_get_():373 variable sar_threshold= libfabric:4137:1704470463::shm:core:fi_param_get_():373 variable tx_size= libfabric:4137:1704470463::shm:core:fi_param_get_():373 variable rx_size= libfabric:4137:1704470463::shm:core:fi_param_get_():373 variable disable_cma= libfabric:4137:1704470463::shm:core:fi_param_get_():373 variable use_dsa_sar= libfabric:4137:1704470463::core:core:ofi_register_provider():504 registering provider: shm (119.0) libfabric:4137:1704470463::core:core:ofi_register_provider():531 "shm" filtered by provider include/exclude list, skipping libfabric:4137:1704470463::core:core:ofi_register_provider():504 registering provider: sm2 (119.0) libfabric:4137:1704470463::core:core:ofi_register_provider():531 "sm2" filtered by provider include/exclude list, skipping libfabric:4137:1704470463::ofi_rxm:core:fi_param_get_():373 variable enable_passthru= libfabric:4137:1704470463::ofi_rxm:core:fi_param_get_():373 variable buffer_size= libfabric:4137:1704470463::ofi_rxm:core:fi_param_get_():373 variable tx_size= libfabric:4137:1704470463::ofi_rxm:core:fi_param_get_():373 variable rx_size= libfabric:4137:1704470463::ofi_rxm:core:fi_param_get_():373 variable msg_tx_size= libfabric:4137:1704470463::ofi_rxm:core:fi_param_get_():373 variable msg_rx_size= libfabric:4137:1704470463::ofi_rxm:core:fi_param_get_():373 variable cm_progress_interval= libfabric:4137:1704470463::ofi_rxm:core:fi_param_get_():373 variable cq_eq_fairness= libfabric:4137:1704470463::ofi_rxm:core:fi_param_get_():373 variable data_auto_progrlibfabric:4117:1704470463::core:core:fi_param_get_():373 variable perf_cntr= libfabric:4117:1704470463::core:core:fi_param_get_():373 variable hook= libfabric:4117:1704470463::core:core:cuda_hmem_dl_init():431 Failed to dlopen libcudart.so libfabric:4117:1704470463::core:core:ofi_hmem_init():444 Hmem iface FI_HMEM_CUDA not supported libfabric:4117:1704470463::core:core:ofi_hmem_init():444 Hmem iface FI_HMEM_ROCR not supported libfabric:4117:1704470463::core:core:ofi_hmem_init():444 Hmem iface FI_HMEM_ZE not supported libfabric:4117:1704470463::core:core:neuron_dl_init():66 Failed to dlopen libnrt.so.1 libfabric:4117:1704470463::core:core:ofi_hmem_init():444 Hmem iface FI_HMEM_NEURON not supported libfabric:4117:1704470463::core:core:synapseai_dl_init():60 Failed to dlopen libSynapse.so libfabric:4117:1704470463::core:core:ofi_hmem_init():444 Hmem iface FI_HMEM_SYNAPSEAI not supported libfabric:4117:1704470463::core:core:fi_param_get_():373 variable hmem_disable_p2p= libfabric:4117:1704470463::core:core:fi_param_get_():373 variable mr_cache_max_size= libfabric:4117:1704470463::core:core:fi_param_get_():373 variable mr_cache_max_count= libfabric:4117:1704470463::core:core:fi_param_get_():373 variable mr_cache_monitor= libfabric:4117:1704470463::core:core:fi_param_get_():373 variable mr_cuda_cache_monitor_enabled= libfabric:4117:1704470463::core:core:fi_param_get_():373 variable mr_rocr_cache_monitor_enabled= libfabric:4117:1704470463::core:core:fi_param_get_():373 variable mr_ze_cache_monitor_enabled= libfabric:4117:1704470463::core:mr:ofi_default_cache_size():79 default cache size=1373431324 libfabric:4117:1704470463::core:core:fi_param_get_():382 read string var provider=EFA libfabric:4117:1704470463::core:core:fi_param_get_():373 variable universe_size= libfabric:4117:1704470463::core:core:fi_param_get_():373 variable av_remove_cleanup= libfabric:4117:1704470463::core:core:fi_param_get_():373 variable offload_coll_provider= libfabric:4117:1704470463::core:core:fi_param_get_():373 variable provider_path= libfabric:4117:1704470463::shm:core:fi_param_get_():373 variable sar_threshold= libfabric:4117:1704470463::shm:core:fi_param_get_():373 variable tx_size= libfabric:4117:1704470463::shm:core:fi_param_get_():373 variable rx_size= libfabric:4117:1704470463::shm:core:fi_param_get_():373 variable disable_cma= libfabric:4117:1704470463::shm:core:fi_param_get_():373 variable use_dsa_sar= libfabric:4117:1704470463::core:core:ofi_register_provider():504 registering provider: shm (119.0) libfabric:4117:1704470463::core:core:ofi_register_provider():531 "shm" filtered by provider include/exclude list, skipping libfabric:4117:1704470463::core:core:ofi_register_provider():504 registering provider: sm2 (119.0) libfabric:4117:1704470463::core:core:ofi_register_provider():531 "sm2" filtered by provider include/exclude list, skipping libfabric:4117:1704470463::ofi_rxm:core:fi_param_get_():373 variable enable_passthru= libfabric:4117:1704470463::ofi_rxm:core:fi_param_get_():373 variable buffer_size= libfabric:4117:1704470463::ofi_rxm:core:fi_param_get_():373 variable tx_size= libfabric:4117:1704470463::ofi_rxm:core:fi_param_get_():373 variable rx_size= libfabric:4117:1704470463::ofi_rxm:core:fi_param_get_():373 variable msg_tx_size= libfabric:4117:1704470463::ofi_rxm:core:fi_param_get_():373 variable msg_rx_size= libfabric:4117:1704470463::ofi_rxm:core:fi_param_get_():373 variable cm_progress_interval= libfabric:4117:1704470463::ofi_rxm:core:fi_param_get_():373 variable cq_eq_fairness= libfabric:4117:1704470463::ofi_rxm:core:fi_param_get_():373 variable data_auto_progress= libfabric:4137:1704470463::ofi_rxm:core:fi_param_get_():373 variable use_rndv_write= libfabric:4137:1704470463::ofi_rxm:core:fi_param_get_():373 variable def_wait_obj= libfabric:4137:1704470463::ofi_rxm:core:fi_param_get_():373 variable def_tcp_wait_obj= libfabric:4137:1704470463::core:core:ofi_register_provider():504 registering provider: ofi_rxm (119.0) libfabric:4137:1704470463::ofi_mrail:core:fi_param_get_():373 variable config= libfabric:4137:1704470463::ofi_mrail:core:fi_param_get_():373 variable addr= libfabric:4137:1704470463::ofi_mrail:core:fi_param_get_():373 variable addr_strc= libfabric:4137:1704470463::ofi_mrail:core:mrail_parse_env_vars():116 unable to read FI_OFI_MRAIL_ADDR env variable libfabric:4137:1704470463::core:core:ofi_register_provider():504 registering provider: ofi_mrail (119.0) ess= libfabric:4117:1704470463::ofi_rxm:core:fi_param_get_():373 variable use_rndv_write= libfabric:4117:1704470463::ofi_rxm:core:fi_param_get_():373 variable def_wait_obj= libfabric:4117:1704470463::ofi_rxm:core:fi_param_get_():373 variable def_tcp_wait_obj= libfabric:4117:1704470463::core:core:ofi_register_provider():504 registering provider: ofi_rxm (119.0) libfabric:4117:1704470463::ofi_mrail:core:fi_param_get_():373 variable config= libfabric:4117:1704470463::ofi_mrail:core:fi_param_get_():373 variable addr= libfabric:4117:1704470463::ofi_mrail:core:fi_param_get_():373 variable addr_strc= libfabric:4117:1704470463::ofi_mrail:core:mrail_parse_env_vars():116 unable to read FI_OFI_MRAIL_ADDR env variable libfabric:4117:1704470463::core:core:ofi_register_provider():504 registering provider: ofi_mrail (119.0) libfabric:4137:1704470463::efa:core:fi_param_get_():373 variable tx_min_credits= libfabric:4137:1704470463::efa:core:fi_param_get_():373 variable tx_queue_size= libfabric:4137:1704470463::efa:core:fi_param_get_():373 variable enable_shm_transfer= libfabric:4137:1704470463::efa:core:fi_param_get_():373 variable use_zcpy_rx= libfabric:4137:1704470463::efa:core:fi_param_get_():373 variable zcpy_rx_seed= libfabric:4137:1704470463::efa:core:fi_param_get_():373 variable shm_av_size= libfabric:4137:1704470463::efa:core:fi_param_get_():373 variable recvwin_size= libfabric:4137:1704470463::efa:core:fi_param_get_():373 variable readcopy_pool_size= libfabric:4137:1704470463::efa:core:fi_param_get_():373 variable cq_size= libfabric:4137:1704470463::efa:core:fi_param_get_():373 variable max_memcpy_size= libfabric:4137:1704470463::efa:core:fi_param_get_():373 variable mr_cache_enable= libfabric:4137:1704470463::efa:core:fi_param_get_():373 variable mr_max_cached_count= libfabric:4137:1704470463::efa:core:fi_param_get_():373 variable mr_max_cached_size= libfabric:4137:1704470463::efa:core:fi_param_get_():373 variable tx_size= libfabric:4137:1704470463::efa:core:fi_param_get_():373 variable rx_size= libfabric:4137:1704470463::efa:core:fi_param_get_():373 variable rx_copy_unexp= libfabric:4137:1704470463::efa:core:fi_param_get_():373 variable rx_copy_ooo= libfabric:4137:1704470463::efa:core:fi_param_get_():373 variable max_timeout= libfabric:4137:1704470463::efa:core:fi_param_get_():373 variable timeout_interval= libfabric:4137:1704470463::efa:core:fi_param_get_():373 variable efa_cq_read_size= libfabric:4137:1704470463::efa:core:fi_param_get_():373 variable shm_cq_read_size= libfabric:4137:1704470463::efa:core:fi_param_get_():373 variable inter_read_segment_size= libfabric:4137:1704470463::efa:core:fi_param_get_():373 variable inter_max_gdrcopy_message_size= libfabric:4137:1704470463::efa:core:fi_param_get_():373 variable use_sm2= libfabric:4137:1704470463::efa:core:fi_param_get_():373 variable use_huge_page= libfabric:4137:1704470463::core:core:ofi_register_provider():504 registering provider: efa (119.0) libfabric:4137:1704470463::core:core:ofi_register_provider():504 registering provider: udp (119.0) libfabric:4137:1704470463::core:core:ofi_register_provider():531 "udp" filtered by provider include/exclude list, skipping libfabric:4137:1704470463::core:core:ofi_register_provider():504 registering provider: sockets (119.0) libfabric:4137:1704470463::core:core:ofi_register_provider():531 "sockets" filtered by provider include/exclude list, skipping libfabric:4137:1704470463::tcp:core:fi_param_get_():373 variable prov_name= libfabric:4137:1704470463::tcp:core:fi_param_get_():373 variable port_high_range= libfabric:4137:1704470463::tcp:core:fi_param_get_():373 variable port_low_range= libfabric:4137:1704470463::tcp:core:fi_param_get_():373 variable tx_size= libfabric:4137:1704470463::tcp:core:fi_param_get_():373 variable rx_size= libfabric:4137:1704470463::tcp:core:fi_param_get_():373 variable max_inject= libfabric:4137:1704470463::tcp:core:fi_param_get_():373 variable max_saved= libfabric:4137:1704470463::tcp:core:fi_param_get_():373 variable max_saved_size= libfabric:4137:1704470463::tcp:core:fi_param_get_():373 variable max_rx_size= libfabric:4137:1704470463::tcp:core:fi_param_get_():373 variable nodelay= libfabric:4137:1704470463::tcp:core:fi_param_get_():373 variable staging_sbuf_size= libfabric:4137:1704470463::tcp:core:fi_param_get_():373 variable prefetch_rbuf_sizelibfabric:4117:1704470463::efa:core:fi_param_get_():373 variable tx_min_credits= libfabric:4117:1704470463::efa:core:fi_param_get_():373 variable tx_queue_size= libfabric:4117:1704470463::efa:core:fi_param_get_():373 variable enable_shm_transfer= libfabric:4117:1704470463::efa:core:fi_param_get_():373 variable use_zcpy_rx= libfabric:4117:1704470463::efa:core:fi_param_get_():373 variable zcpy_rx_seed= libfabric:4117:1704470463::efa:core:fi_param_get_():373 variable shm_av_size= libfabric:4117:1704470463::efa:core:fi_param_get_():373 variable recvwin_size= libfabric:4117:1704470463::efa:core:fi_param_get_():373 variable readcopy_pool_size= libfabric:4117:1704470463::efa:core:fi_param_get_():373 variable cq_size= libfabric:4117:1704470463::efa:core:fi_param_get_():373 variable max_memcpy_size= libfabric:4117:1704470463::efa:core:fi_param_get_():373 variable mr_cache_enable= libfabric:4117:1704470463::efa:core:fi_param_get_():373 variable mr_max_cached_count= libfabric:4117:1704470463::efa:core:fi_param_get_():373 variable mr_max_cached_size= libfabric:4117:1704470463::efa:core:fi_param_get_():373 variable tx_size= libfabric:4117:1704470463::efa:core:fi_param_get_():373 variable rx_size= libfabric:4117:1704470463::efa:core:fi_param_get_():373 variable rx_copy_unexp= libfabric:4117:1704470463::efa:core:fi_param_get_():373 variable rx_copy_ooo= libfabric:4117:1704470463::efa:core:fi_param_get_():373 variable max_timeout= libfabric:4117:1704470463::efa:core:fi_param_get_():373 variable timeout_interval= libfabric:4117:1704470463::efa:core:fi_param_get_():373 variable efa_cq_read_size= libfabric:4117:1704470463::efa:core:fi_param_get_():373 variable shm_cq_read_size= libfabric:4117:1704470463::efa:core:fi_param_get_():373 variable inter_read_segment_size= libfabric:4117:1704470463::efa:core:fi_param_get_():373 variable inter_max_gdrcopy_message_size= libfabric:4117:1704470463::efa:core:fi_param_get_():373 variable use_sm2= libfabric:4117:1704470463::efa:core:fi_param_get_():373 variable use_huge_page= libfabric:4117:1704470463::core:core:ofi_register_provider():504 registering provider: efa (119.0) libfabric:4117:1704470463::core:core:ofi_register_provider():504 registering provider: udp (119.0) libfabric:4117:1704470463::core:core:ofi_register_provider():531 "udp" filtered by provider include/exclude list, skipping libfabric:4117:1704470463::core:core:ofi_register_provider():504 registering provider: sockets (119.0) libfabric:4117:1704470463::core:core:ofi_register_provider():531 "sockets" filtered by provider include/exclude list, skipping libfabric:4117:1704470463::tcp:core:fi_param_get_():373 variable prov_name= libfabric:4117:1704470463::tcp:core:fi_param_get_():373 variable port_high_range= libfabric:4117:1704470463::tcp:core:fi_param_get_():373 variable port_low_range= libfabric:4117:1704470463::tcp:core:fi_param_get_():373 variable tx_size= libfabric:4117:1704470463::tcp:core:fi_param_get_():373 variable rx_size= libfabric:4117:1704470463::tcp:core:fi_param_get_():373 variable max_inject= libfabric:4117:1704470463::tcp:core:fi_param_get_():373 variable max_saved= libfabric:4117:1704470463::tcp:core:fi_param_get_():373 variable max_saved_size= libfabric:4117:1704470463::tcp:core:fi_param_get_():373 variable max_rx_size= libfabric:4117:1704470463::tcp:core:fi_param_get_():373 variable nodelay= libfabric:4117:1704470463::tcp:core:fi_param_get_():373 variable staging_sbuf_size= libfabric:4117:1704470463::tcp:core:fi_param_get_():373 variable prefetch_rbuf_size= libfabric:4137:1704470463::tcp:core:fi_param_get_():373 variable zerocopy_size= libfabric:4137:1704470463::tcp:core:fi_param_get_():373 variable trace_msg= libfabric:4137:1704470463::tcp:core:fi_param_get_():373 variable disable_auto_progress= libfabric:4137:1704470463::tcp:core:fi_param_get_():373 variable io_uring= libfabric:4137:1704470463::core:core:ofi_register_provider():504 registering provider: tcp (119.0) libfabric:4137:1704470463::core:core:ofi_register_provider():531 "tcp" filtered by provider include/exclude list, skipping libfabric:4137:1704470463::core:core:ofi_register_provider():504 registering provider: ofi_hook_perf (119.0) libfabric:4137:1704470463::core:core:ofi_register_provider():504 registering provider: ofi_hook_trace (119.0) libfabric:4137:1704470463::core:core:ofi_register_provider():504 registering provider: ofi_hook_profile (119.0) libfabric:4137:1704470463::core:core:ofi_register_provider():504 registering provider: ofi_hook_debug (119.0) libfabric:4137:1704470463::core:core:cuda_hmem_dl_init():431 Failed to dlopen libcudart.so libfabric:4137:1704470463::core:core:ofi_hmem_init():444 Hmem iface FI_HMEM_CUDA not supported libfabric:4137:1704470463::core:core:ofi_hmem_init():444 Hmem iface FI_HMEM_ROCR not supported libfabric:4137:1704470463::core:core:ofi_hmem_init():444 Hmem iface FI_HMEM_ZE not supported libfabric:4137:1704470463::core:core:neuron_dl_init():66 Failed to dlopen libnrt.so.1 libfabric:4137:1704470463::core:core:ofi_hmem_init():444 Hmem iface FI_HMEM_NEURON not supported libfabric:4137:1704470463::core:core:synapseai_dl_init():60 Failed to dlopen libSynapse.so libfabric:4137:1704470463::core:core:ofi_hmem_init():444 Hmem iface FI_HMEM_SYNAPSEAI not supported libfabric:4137:1704470463::core:core:fi_param_get_():373 variable hmem_disable_p2p= libfabric:4137:1704470463::core:core:ofi_register_provider():504 registering provider: ofi_hook_hmem (119.0) libfabric:4137:1704470463::core:core:ofi_register_provider():504 registering provider: ofi_hook_dmabuf_peer_mem (119.0) libfabric:4137:1704470463::core:core:ofi_register_provider():504 registering provider: ofi_hook_noop (119.0) libfabric:4137:1704470463::core:core:ofi_register_provider():504 registering provider: off_coll (119.0) libfabric:4137:1704470463::efa:core:ofi_check_ep_type():667 unsupported endpoint type libfabric:4137:1704470463::efa:core:ofi_check_ep_type():668 Supported: FI_EP_DGRAM libfabric:4137:1704470463::efa:core:ofi_check_ep_type():668 Requested: FI_EP_RDM libfabric:4137:1704470463::ofi_rxm:core:fi_param_get_():373 variable use_srx= libfabric:4137:1704470463:ofi_rxm:core:core:ofi_layering_ok():1168 Provider ofi_rxm is excluded libfabric:4137:1704470463:ofi_rxm:core:core:ofi_layering_ok():1179 Need core provider, skipping ofi_mrail libfabric:4137:1704470463::ofi_rxm:core:fi_param_get_():373 variable use_srx= libfabric:4137:1704470463:ofi_rxm:core:core:ofi_layering_ok():1168 Provider ofi_rxm is excluded libfabric:4137:1704470463:ofi_rxm:core:core:ofi_layering_ok():1179 Need core provider, skipping ofi_mrail libfabric:4137:1704470463::ofi_rxm:core:fi_param_get_():373 variable use_srx= libfabric:4137:1704470463:ofi_rxm:core:core:ofi_layering_ok():1185 Skipping util;efa layering libfabric:4137:1704470463:ofi_rxm:core:core:ofi_layering_ok():1179 Need core provider, skipping ofi_rxm libfabric:4137:1704470463:ofi_rxm:core:core:ofi_layering_ok():1179 Need core provider, skipping ofi_mrail libfabric:4137:1704470463::ofi_rxm:core:fi_param_get_():373 variable use_srx= libfabric:4137:1704470463:ofi_rxm:core:core:ofi_layering_ok():1185 Skipping util;efa layering libfabric:4137:1704470463:ofi_rxm:core:core:ofi_layering_ok():1179 Need core provider, skipping ofi_rxm libfabric:4137:1704470463:ofi_rxm:core:core:of= libfabric:4117:1704470463::tcp:core:fi_param_get_():373 variable zerocopy_size= libfabric:4117:1704470463::tcp:core:fi_param_get_():373 variable trace_msg= libfabric:4117:1704470463::tcp:core:fi_param_get_():373 variable disable_auto_progress= libfabric:4117:1704470463::tcp:core:fi_param_get_():373 variable io_uring= libfabric:4117:1704470463::core:core:ofi_register_provider():504 registering provider: tcp (119.0) libfabric:4117:1704470463::core:core:ofi_register_provider():531 "tcp" filtered by provider include/exclude list, skipping libfabric:4117:1704470463::core:core:ofi_register_provider():504 registering provider: ofi_hook_perf (119.0) libfabric:4117:1704470463::core:core:ofi_register_provider():504 registering provider: ofi_hook_trace (119.0) libfabric:4117:1704470463::core:core:ofi_register_provider():504 registering provider: ofi_hook_profile (119.0) libfabric:4117:1704470463::core:core:ofi_register_provider():504 registering provider: ofi_hook_debug (119.0) libfabric:4117:1704470463::core:core:cuda_hmem_dl_init():431 Failed to dlopen libcudart.so libfabric:4117:1704470463::core:core:ofi_hmem_init():444 Hmem iface FI_HMEM_CUDA not supported libfabric:4117:1704470463::core:core:ofi_hmem_init():444 Hmem iface FI_HMEM_ROCR not supported libfabric:4117:1704470463::core:core:ofi_hmem_init():444 Hmem iface FI_HMEM_ZE not supported libfabric:4117:1704470463::core:core:neuron_dl_init():66 Failed to dlopen libnrt.so.1 libfabric:4117:1704470463::core:core:ofi_hmem_init():444 Hmem iface FI_HMEM_NEURON not supported libfabric:4117:1704470463::core:core:synapseai_dl_init():60 Failed to dlopen libSynapse.so libfabric:4117:1704470463::core:core:ofi_hmem_init():444 Hmem iface FI_HMEM_SYNAPSEAI not supported libfabric:4117:1704470463::core:core:fi_param_get_():373 variable hmem_disable_p2p= libfabric:4117:1704470463::core:core:ofi_register_provider():504 registering provider: ofi_hook_hmem (119.0) libfabric:4117:1704470463::core:core:ofi_register_provider():504 registering provider: ofi_hook_dmabuf_peer_mem (119.0) libfabric:4117:1704470463::core:core:ofi_register_provider():504 registering provider: ofi_hook_noop (119.0) libfabric:4117:1704470463::core:core:ofi_register_provider():504 registering provider: off_coll (119.0) libfabric:4117:1704470463::efa:core:ofi_check_ep_type():667 unsupported endpoint type libfabric:4117:1704470463::efa:core:ofi_check_ep_type():668 Supported: FI_EP_DGRAM libfabric:4117:1704470463::efa:core:ofi_check_ep_type():668 Requested: FI_EP_RDM libfabric:4117:1704470463::ofi_rxm:core:fi_param_get_():373 variable use_srx= libfabric:4117:1704470463:ofi_rxm:core:core:ofi_layering_ok():1168 Provider ofi_rxm is excluded libfabric:4117:1704470463:ofi_rxm:core:core:ofi_layering_ok():1179 Need core provider, skipping ofi_mrail libfabric:4117:1704470463::ofi_rxm:core:fi_param_get_():373 variable use_srx= libfabric:4117:1704470463:ofi_rxm:core:core:ofi_layering_ok():1168 Provider ofi_rxm is excluded libfabric:4117:1704470463:ofi_rxm:core:core:ofi_layering_ok():1179 Need core provider, skipping ofi_mrail libfabric:4117:1704470463::ofi_rxm:core:fi_param_get_():373 variable use_srx= libfabric:4117:1704470463:ofi_rxm:core:core:ofi_layering_ok():1185 Skipping util;efa layering libfabric:4117:1704470463:ofi_rxm:core:core:ofi_layering_ok():1179 Need core provider, skipping ofi_rxm libfabric:4117:1704470463:ofi_rxm:core:core:ofi_layering_ok():1179 Need core provider, skipping ofi_mrail libfabric:4117:1704470463::ofi_rxm:core:fi_param_get_():373 variable use_srx= libfabric:4117:1704470463:ofi_rxm:core:core:ofi_layering_ok():1185 Skipping util;efa layering libfabric:4117:1704470463:ofi_rxm:core:core:ofi_layering_ok():1179 Need core provider, skipping ofi_rxm libfabric:4117:1704470463:ofi_rxm:core:core:ofi_layering_ok():1179 Need core provider, skipping ofi_mrail libfabric:4137:1704470463::core:core:fi_getinfo_():1289 fi_getinfo: provider ofi_rxm returned -61 (No data available) libfabric:4137:1704470463::ofi_mrail:fabric:mrail_get_core_info():286 OFI_MRAIL_ADDR_STRC env variable not set! libfabric:4137:1704470463::core:core:fi_getinfo_():1289 fi_getinfo: provider ofi_mrail returned -61 (No data available) i_layering_ok():1179 Need core provider, skipping ofi_mrail libfabric:4117:1704470463::core:core:fi_getinfo_():1289 fi_getinfo: provider ofi_rxm returned -61 (No data available) libfabric:4117:1704470463::ofi_mrail:fabric:mrail_get_core_info():286 OFI_MRAIL_ADDR_STRC env variable not set! libfabric:4117:1704470463::core:core:fi_getinfo_():1289 fi_getinfo: provider ofi_mrail returned -61 (No data available) libfabric:4137:1704470463::core:mr:ofi_monitor_import():847 setting imported memory monitor as default libfabric:4137:1704470463::core:core:fi_fabric_():1489 Opened fabric: efa libfabric:4137:1704470463::efa:domain:efa_domain_init_device_and_pd():121 Domain rdmap0s6-rdm selected device rdmap0s6 libfabric:4137:1704470463::efa:domain:efa_mr_cache_open():176 EFA MR cache enabled, max_cnt: 235929 max_size: 92771293593 libfabric:4137:1704470463::core:core:ofi_layering_ok():1205 Skipping shm;ofi_rxm layering libfabric:4137:1704470463::shm:core:ofi_check_info():1060 Unsupported capabilities libfabric:4137:1704470463::shm:core:ofi_check_info():1061 Supported: FI_MSG, FI_RMA, FI_TAGGED, FI_READ, FI_WRITE, FI_RECV, FI_SEND, FI_REMOTE_READ, FI_REMOTE_WRITE, FI_MULTI_RECV, FI_LOCAL_COMM, FI_RMA_EVENT, FI_SOURCE, FI_DIRECTED_RECV, FI_HMEM libfabric:4137:1704470463::shm:core:ofi_check_info():1061 Requested: FI_RMA, FI_ATOMIC, FI_READ, FI_WRITE, FI_REMOTE_READ, FI_REMOTE_WRITE, FI_MULTI_RECV libfabric:4137:1704470463::core:core:ofi_layering_ok():1205 Skipping shm;ofi_mrail layering libfabric:4137:1704470463::core:core:fi_fabric_():1489 Opened fabric: shm libfabric:4137:1704470463::shm:core:ofi_check_rx_attr():805 Tx only caps ignored in Rx caps libfabric:4137:1704470463::shm:core:ofi_check_tx_attr():903 Rx only caps ignored in Tx caps libfabric:4137:1704470463::shm:core:ofi_check_info():1060 Unsupported capabilities libfabric:4137:1704470463::shm:core:ofi_check_info():1061 Supported: FI_MSG, FI_RMA, FI_TAGGED, FI_READ, FI_WRITE, FI_RECV, FI_SEND, FI_REMOTE_READ, FI_REMOTE_WRITE, FI_MULTI_RECV, FI_LOCAL_COMM, FI_RMA_EVENT, FI_SOURCE, FI_DIRECTED_RECV, FI_HMEM libfabric:4137:1704470463::shm:core:ofi_check_info():1061 Requested: FI_RMA, FI_ATOMIC, FI_READ, FI_WRITE, FI_REMOTE_READ, FI_REMOTE_WRITE, FI_MULTI_RECV, FI_RMA_EVENT libfabric:4137:1704470463::efa:core:fi_param_get_():373 variable runt_size= libfabric:4137:1704470463::efa:core:fi_param_get_():373 variable inter_max_medium_message_size= libfabric:4137:1704470463::efa:core:fi_param_get_():373 variable inter_min_read_message_size= libfabric:4137:1704470463::efa:core:fi_param_get_():373 variable inter_min_read_write_size= libfabric:4137:1704470463::efa:domain:efa_domain_hmem_info_init_cuda():153 FI_HMEM_CUDA is not initialized libfabric:4137:1704470463::efa:domain:efa_domain_hmem_info_init_neuron():230 FI_HMEM_NEURON is not initialized libfabric:4137:1704470463::efa:domain:efa_domain_hmem_info_init_synapseai():316 FI_HMEM_SYNAPSEAI is not initialized libfabric:4137:1704470463::core:core:fi_param_get_():373 variable universe_size= libfabric:4137:1704470463::efa:av:util_av_init():486 AV size 16384 libfabric:4137:1704470463::shm:av:util_av_init():486 AV size 256 libfabric:4137:1704470463::efa:av:efa_av_open():983 fi_av_attr:0 libfabric:4137:1704470463::efa:core:ofi_check_rx_attr():805 Tx only caps ignored in Rx caps libfabric:4137:1704470463::efa:core:ofi_check_tx_attr():903 Rx only caps ignored in Tx caps libfabric:4137:1704470463::efa:core:ofi_check_ep_type():667 unsupported endpoint type libfabric:4137:1704470463::efa:core:ofi_check_ep_type():668 Supported: FI_EP_DGRAM libfabric:4137:1704470463::efa:core:ofi_check_ep_type():668 Requested: FI_EP_RDM libfabric:4137:1704470463::shm:core:ofi_check_rx_attr():805 Tx only caps ignored in Rx caps libfabric:4137:1704470463::shm:core:ofi_check_tx_attr():903 Rx only caps ignored in Tx caps libfabric:4137:1704470463::shm:core:ofi_check_info():1060 Unsupported capabilities libfabric:4137:1704470463::shm:core:ofi_check_info():1061 Supported: FI_MSG, FI_RMA, FI_TAGGED, FI_READ, FI_WRITE, FI_RECV, FI_SEND, FI_REMOTE_READ, FI_REMOTE_WRITE, FI_MULTI_RECV, FI_LOCAL_COMM, FI_RMA_EVENT, FI_SOURCE, FI_DIRECTED_RECV, FI_HMEM libfabric:4137:1704470463::shm:core:ofi_check_info():1061 Requested: FI_RMA,libfabric:4117:1704470463::core:mr:ofi_monitor_import():847 setting imported memory monitor as default libfabric:4117:1704470463::core:core:fi_fabric_():1489 Opened fabric: efa libfabric:4117:1704470463::efa:domain:efa_domain_init_device_and_pd():121 Domain rdmap0s6-rdm selected device rdmap0s6 libfabric:4117:1704470463::efa:domain:efa_mr_cache_open():176 EFA MR cache enabled, max_cnt: 235929 max_size: 92771293593 libfabric:4117:1704470463::core:core:ofi_layering_ok():1205 Skipping shm;ofi_rxm layering libfabric:4117:1704470463::shm:core:ofi_check_info():1060 Unsupported capabilities libfabric:4117:1704470463::shm:core:ofi_check_info():1061 Supported: FI_MSG, FI_RMA, FI_TAGGED, FI_READ, FI_WRITE, FI_RECV, FI_SEND, FI_REMOTE_READ, FI_REMOTE_WRITE, FI_MULTI_RECV, FI_LOCAL_COMM, FI_RMA_EVENT, FI_SOURCE, FI_DIRECTED_RECV, FI_HMEM libfabric:4117:1704470463::shm:core:ofi_check_info():1061 Requested: FI_RMA, FI_ATOMIC, FI_READ, FI_WRITE, FI_REMOTE_READ, FI_REMOTE_WRITE, FI_MULTI_RECV libfabric:4117:1704470463::core:core:ofi_layering_ok():1205 Skipping shm;ofi_mrail layering libfabric:4117:1704470463::core:core:fi_fabric_():1489 Opened fabric: shm libfabric:4117:1704470463::shm:core:ofi_check_rx_attr():805 Tx only caps ignored in Rx caps libfabric:4117:1704470463::shm:core:ofi_check_tx_attr():903 Rx only caps ignored in Tx caps libfabric:4117:1704470463::shm:core:ofi_check_info():1060 Unsupported capabilities libfabric:4117:1704470463::shm:core:ofi_check_info():1061 Supported: FI_MSG, FI_RMA, FI_TAGGED, FI_READ, FI_WRITE, FI_RECV, FI_SEND, FI_REMOTE_READ, FI_REMOTE_WRITE, FI_MULTI_RECV, FI_LOCAL_COMM, FI_RMA_EVENT, FI_SOURCE, FI_DIRECTED_RECV, FI_HMEM libfabric:4117:1704470463::shm:core:ofi_check_info():1061 Requested: FI_RMA, FI_ATOMIC, FI_READ, FI_WRITE, FI_REMOTE_READ, FI_REMOTE_WRITE, FI_MULTI_RECV, FI_RMA_EVENT libfabric:4117:1704470463::efa:core:fi_param_get_():373 variable runt_size= libfabric:4117:1704470463::efa:core:fi_param_get_():373 variable inter_max_medium_message_size= libfabric:4117:1704470463::efa:core:fi_param_get_():373 variable inter_min_read_message_size= libfabric:4117:1704470463::efa:core:fi_param_get_():373 variable inter_min_read_write_size= libfabric:4117:1704470463::efa:domain:efa_domain_hmem_info_init_cuda():153 FI_HMEM_CUDA is not initialized libfabric:4117:1704470463::efa:domain:efa_domain_hmem_info_init_neuron():230 FI_HMEM_NEURON is not initialized libfabric:4117:1704470463::efa:domain:efa_domain_hmem_info_init_synapseai():316 FI_HMEM_SYNAPSEAI is not initialized libfabric:4117:1704470463::core:core:fi_param_get_():373 variable universe_size= libfabric:4117:1704470463::efa:av:util_av_init():486 AV size 16384 libfabric:4117:1704470463::shm:av:util_av_init():486 AV size 256 libfabric:4117:1704470463::efa:av:efa_av_open():983 fi_av_attr:0 libfabric:4117:1704470463::efa:core:ofi_check_rx_attr():805 Tx only caps ignored in Rx caps libfabric:4117:1704470463::efa:core:ofi_check_tx_attr():903 Rx only caps ignored in Tx caps libfabric:4117:1704470463::efa:core:ofi_check_ep_type():667 unsupported endpoint type libfabric:4117:1704470463::efa:core:ofi_check_ep_type():668 Supported: FI_EP_DGRAM libfabric:4117:1704470463::efa:core:ofi_check_ep_type():668 Requested: FI_EP_RDM libfabric:4117:1704470463::shm:core:ofi_check_rx_attr():805 Tx only caps ignored in Rx caps libfabric:4117:1704470463::shm:core:ofi_check_tx_attr():903 Rx only caps ignored in Tx caps libfabric:4117:1704470463::shm:core:ofi_check_info():1060 Unsupported capabilities libfabric:4117:1704470463::shm:core:ofi_check_info():1061 Supported: FI_MSG, FI_RMA, FI_TAGGED, FI_READ, FI_WRITE, FI_RECV, FI_SEND, FI_REMOTE_READ, FI_REMOTE_WRITE, FI_MULTI_RECV, FI_LOCAL_COMM, FI_RMA_EVENT, FI_SOURCE, FI_DIRECTED_RECV, FI_HMEM libfabric:4117:1704470463::shm:core:ofi_check_info():1061 Requested: FI_RMA, FI_ATOMIC, FI_READ, FI_WRITE, FI_REMOTE_READ, FI_REMOTE_WRITE, FI_MULTI_RECV, FI_RMA_EVENT libfabric:4137:1704470463::efa:ep_ctrl:efa_rdm_ep_open():452 efa_rdm_ep->host_id: i-0ef5266ddb677e20a libfabric:4137:1704470463::efa:core:fi_param_get_():373 variable use_device_rdma= libfabric:4137:1704470463::efa:cq:efa_rdm_ep_open():469 FI_DELIVERY_COMPLETE unsupported libfabric:4137:1704470463::efa:ep_ctrl:efa_rdm_ep_set_use_zcpy_rx():400 efa_rdm_ep->use_zcpy_rx = 0 libfabric:4137:1704470463::efa:ep_ctrl:efa_rdm_ep_open():490 efa_rdm_ep->handle_resource_management = 2 FI_ATOMIC, FI_READ, FI_WRITE, FI_REMOTE_READ, FI_REMOTE_WRITE, FI_MULTI_RECV, FI_RMA_EVENT libfabric:4117:1704470463::efa:ep_ctrl:efa_rdm_ep_open():452 efa_rdm_ep->host_id: i-0339cefba46aa889a libfabric:4117:1704470463::efa:core:fi_param_get_():373 variable use_device_rdma= libfabric:4117:1704470463::efa:cq:efa_rdm_ep_open():469 FI_DELIVERY_COMPLETE unsupported libfabric:4117:1704470463::efa:ep_ctrl:efa_rdm_ep_set_use_zcpy_rx():400 efa_rdm_ep->use_zcpy_rx = 0 libfabric:4117:1704470463::efa:ep_ctrl:efa_rdm_ep_open():490 efa_rdm_ep->handle_resource_management = 2 libfabric:4117:1704470463::efa:ep_ctrl:efa_base_ep_enable_qp():230 QP enabled! qp_n: 0 qkey: 285354193 libfabric:4137:1704470463::efa:ep_ctrl:efa_base_ep_enable_qp():230 QP enabled! qp_n: 0 qkey: 1815864927 libfabric:4137:1704470463::efa:ep_ctrl:efa_rdm_ep_ctrl():1056 libfabric 1.19.0amzn4.0 efa endpoint created! address: fi_addr_efa://[fe80::41c:4ff:fe4f:9f91]:0:1815864927 libfabric:4137:1704470463::efa:ep_ctrl:efa_base_ep_getname():324 EP addr: GID[fe80::41c:4ff:fe4f:9f91] QP[0] QKEY[1815864927] (length 64) libfabric:4117:1704470463::efa:ep_ctrl:efa_rdm_ep_ctrl():1056 libfabric 1.19.0amzn4.0 efa endpoint created! address: fi_addr_efa://[fe80::408:67ff:fe14:3559]:0:285354193 libfabric:4117:1704470463::efa:ep_ctrl:efa_base_ep_getname():324 EP addr: GID[fe80::408:67ff:fe14:3559] QP[0] QKEY[285354193] (length 64) libfabric:4137:1704470463::efa:core:ofi_check_ep_type():667 unsupported endpoint type libfabric:4137:1704470463::efa:core:ofi_check_ep_type():668 Supported: FI_EP_DGRAM libfabric:4137:1704470463::efa:core:ofi_check_ep_type():668 Requested: FI_EP_RDM libfabric:4137:1704470463::core:core:ofi_layering_ok():1205 Skipping efa;ofi_rxm layering libfabric:4137:1704470463::core:core:ofi_layering_ok():1205 Skipping efa;ofi_mrail layering libfabric:4137:1704470463::core:core:fi_fabric_():1489 Opened fabric: efa libfabric:4137:1704470463::efa:domain:efa_domain_init_device_and_pd():121 Domain rdmap0s6-rdm selected device rdmap0s6 libfabric:4137:1704470463::efa:domain:efa_mr_cache_open():176 EFA MR cache enabled, max_cnt: 235929 max_size: 92771293593 libfabric:4137:1704470463::core:core:ofi_layering_ok():1205 Skipping shm;ofi_rxm layering libfabric:4137:1704470463::core:core:ofi_layering_ok():1205 Skipping shm;ofi_mrail layering libfabric:4137:1704470463::core:core:fi_fabric_():1489 Opened fabric: shm libfabric:4137:1704470463::shm:core:ofi_check_rx_attr():805 Tx only caps ignored in Rx caps libfabric:4137:1704470463::shm:core:ofi_check_tx_attr():903 Rx only caps ignored in Tx caps libfabric:4137:1704470463::shm:core:ofi_check_rx_attr():805 Tx only caps ignored in Rx caps libfabric:4137:1704470463::shm:core:ofi_check_tx_attr():903 Rx only caps ignored in Tx caps libfabric:4137:1704470463::shm:core:ofi_check_tx_attr():940 inject_size too large libfabric:4137:1704470463::shm:core:ofi_check_tx_attr():941 Supported: 0 libfabric:4137:1704470463::shm:core:ofi_check_tx_attr():941 Requested: 4096 libfabric:4137:1704470463::efa:core:fi_param_get_():373 variable runt_size= libfabric:4137:1704470463::efa:core:fi_param_get_():373 variable inter_max_medium_message_size= libfabric:4137:1704470463::efa:core:fi_param_get_():373 variable inter_min_read_message_size= libfabric:4137:1704470463::efa:core:fi_param_get_():373 variable inter_min_read_write_size= libfabric:4137:1704470463::efa:domain:efa_domain_hmem_info_init_cuda():153 FI_HMEM_CUDA is not initialized libfabric:4137:1704470463::efa:domain:efa_domain_hmem_info_init_neuron():230 FI_HMEM_NEURON is not initialized libfabric:4137:1704470463::efa:domain:efa_domain_hmem_info_init_synapseai():316 FI_HMEM_SYNAPSEAI is not initialized libfabric:4137:1704470463::efa:core:ofi_check_rx_attr():805 Tx only caps ignored in Rx caps libfabric:4137:1704470463::efa:core:ofi_check_tx_attr():903 Rx only caps ignored in Tx caps libfabric:4137:1704470463::efa:core:ofi_check_ep_type():667 unsupported endpoint type libfabric:4137:1704470463::efa:core:ofi_check_ep_type():668 Supported: FI_EP_DGRAM libfabric:4137:1704470463::efa:core:ofi_check_ep_type():668 Requested: FI_EP_RDM libfabric:4137:1704470463::shm:core:ofi_check_rx_attr():805 Tx only caps ignored in Rx caps libfabric:4137:1704470463::shm:core:ofi_check_tx_attr():903 Rx only caps ignored in Tx caps libfabric:4137:1704470463::shm:core:ofi_check_rx_attr():805 Tx only caps ignored in Rx caps libfabric:4137:1704470463::shm:core:ofi_check_tx_attr():903 Rx only caps ignored in Tx caps libfabric:4137:1704470463::shm:core:ofi_check_tx_attr():940 inject_size too large libfabric:4137:1704470463::shm:core:ofi_check_tx_attr():941 Supported: 0 libfabric:4137:1704470463::shm:core:ofi_check_tx_attr():941 Requested: 4096 libfabric:4137:1704470463::efa:ep_ctrl:efa_rdm_ep_open():452 efa_rdm_ep->host_id: i-0ef5266ddb677e20a libfabric:4137:1704470463::efa:core:fi_param_get_():373 variable use_device_rdma= libfabric:4137:1704470463::efa:ep_ctrl:efa_rdm_ep_set_use_zcpy_rx():400 efa_rdm_ep->use_zcpy_rx = 0 libfabric:4137:1704470463::efa:ep_ctrl:efa_rdm_ep_open():490 efa_rdm_ep->handle_resource_management = 2 libfabric:4117:1704470463::efa:core:ofi_check_ep_type():667 unsupported endpoint type libfabric:4117:1704470463::efa:core:ofi_check_ep_type():668 Supported: FI_EP_DGRAM libfabric:4117:1704470463::efa:core:ofi_check_ep_type():668 Requested: FI_EP_RDM libfabric:4117:1704470463::core:core:ofi_layering_ok():1205 Skipping efa;ofi_rxm layering libfabric:4117:1704470463::core:core:ofi_layering_ok():1205 Skipping efa;ofi_mrail layering libfabric:4117:1704470463::core:core:fi_fabric_():1489 Opened fabric: efa libfabric:4117:1704470463::efa:domain:efa_domain_init_device_and_pd():121 Domain rdmap0s6-rdm selected device rdmap0s6 libfabric:4117:1704470463::efa:domain:efa_mr_cache_open():176 EFA MR cache enabled, max_cnt: 235929 max_size: 92771293593 libfabric:4117:1704470463::core:core:ofi_layering_ok():1205 Skipping shm;ofi_rxm layering libfabric:4117:1704470463::core:core:ofi_layering_ok():1205 Skipping shm;ofi_mrail layering libfabric:4117:1704470463::core:core:fi_fabric_():1489 Opened fabric: shm libfabric:4117:1704470463::shm:core:ofi_check_rx_attr():805 Tx only caps ignored in Rx caps libfabric:4117:1704470463::shm:core:ofi_check_tx_attr():903 Rx only caps ignored in Tx caps libfabric:4117:1704470463::shm:core:ofi_check_rx_attr():805 Tx only caps ignored in Rx caps libfabric:4117:1704470463::shm:core:ofi_check_tx_attr():903 Rx only caps ignored in Tx caps libfabric:4117:1704470463::shm:core:ofi_check_tx_attr():940 inject_size too large libfabric:4117:1704470463::shm:core:ofi_check_tx_attr():941 Supported: 0 libfabric:4117:1704470463::shm:core:ofi_check_tx_attr():941 Requested: 4096 libfabric:4117:1704470463::efa:core:fi_param_get_():373 variable runt_size= libfabric:4117:1704470463::efa:core:fi_param_get_():373 variable inter_max_medium_message_size= libfabric:4117:1704470463::efa:core:fi_param_get_():373 variable inter_min_read_message_size= libfabric:4117:1704470463::efa:core:fi_param_get_():373 variable inter_min_read_write_size= libfabric:4117:1704470463::efa:domain:efa_domain_hmem_info_init_cuda():153 FI_HMEM_CUDA is not initialized libfabric:4117:1704470463::efa:domain:efa_domain_hmem_info_init_neuron():230 FI_HMEM_NEURON is not initialized libfabric:4117:1704470463::efa:domain:efa_domain_hmem_info_init_synapseai():316 FI_HMEM_SYNAPSEAI is not initialized libfabric:4117:1704470463::efa:core:ofi_check_rx_attr():805 Tx only caps ignored in Rx caps libfabric:4117:1704470463::efa:core:ofi_check_tx_attr():903 Rx only caps ignored in Tx caps libfabric:4117:1704470463::efa:core:ofi_check_ep_type():667 unsupported endpoint type libfabric:4117:1704470463::efa:core:ofi_check_ep_type():668 Supported: FI_EP_DGRAM libfabric:4117:1704470463::efa:core:ofi_check_ep_type():668 Requested: FI_EP_RDM libfabric:4117:1704470463::shm:core:ofi_check_rx_attr():805 Tx only caps ignored in Rx caps libfabric:4117:1704470463::shm:core:ofi_check_tx_attr():903 Rx only caps ignored in Tx caps libfabric:4117:1704470463::shm:core:ofi_check_rx_attr():805 Tx only caps ignored in Rx caps libfabric:4117:1704470463::shm:core:ofi_check_tx_attr():903 Rx only caps ignored in Tx caps libfabric:4117:1704470463::shm:core:ofi_check_tx_attr():940 inject_size too large libfabric:4117:1704470463::shm:core:ofi_check_tx_attr():941 Supported: 0 libfabric:4117:1704470463::shm:core:ofi_check_tx_attr():941 Requested: 4096 libfabric:4117:1704470463::efa:ep_ctrl:efa_rdm_ep_open():452 efa_rdm_ep->host_id: i-0339cefba46aa889a libfabric:4117:1704470463::efa:core:fi_param_get_():373 variable use_device_rdma= libfabric:4117:1704470463::efa:ep_ctrl:efa_rdm_ep_set_use_zcpy_rx():400 efa_rdm_ep->use_zcpy_rx = 0 libfabric:4117:1704470463::efa:ep_ctrl:efa_rdm_ep_open():490 efa_rdm_ep->handle_resource_management = 2 libfabric:4117:1704470463::core:core:fi_param_get_():373 variable universe_size= libfabric:4117:1704470463::efa:av:util_av_init():486 AV size 16384 libfabric:4117:1704470463::shm:av:util_av_init():486 AV size 256 libfabric:4117:1704470463::efa:av:efa_av_open():983 fi_av_attr:0 libfabric:4117:1704470463::efa:ep_ctrl:efa_base_ep_enable_qp():230 QP enabled! qp_n: 1 qkey: 1806222746 libfabric:4137:1704470463::core:core:fi_param_get_():373 variable universe_size= libfabric:4137:1704470463::efa:av:util_av_init():486 AV size 16384 libfabric:4137:1704470463::shm:av:util_av_init():486 AV size 256 libfabric:4137:1704470463::efa:av:efa_av_open():983 fi_av_attr:0 libfabric:4137:1704470463::efa:ep_ctrl:efa_base_ep_enable_qp():230 QP enabled! qp_n: 1 qkey: 1280243318 libfabric:4117:1704470463::efa:ep_ctrl:efa_rdm_ep_ctrl():1056 libfabric 1.19.0amzn4.0 efa endpoint created! address: fi_addr_efa://[fe80::408:67ff:fe14:3559]:1:1806222746 libfabric:4117:1704470463::efa:ep_ctrl:efa_base_ep_getname():324 EP addr: GID[fe80::408:67ff:fe14:3559] QP[1] QKEY[1806222746] (length 64) libfabric:4137:1704470463::efa:ep_ctrl:efa_rdm_ep_ctrl():1056 libfabric 1.19.0amzn4.0 efa endpoint created! address: fi_addr_efa://[fe80::41c:4ff:fe4f:9f91]:1:1280243318 libfabric:4137:1704470463::efa:ep_ctrl:efa_base_ep_getname():324 EP addr: GID[fe80::41c:4ff:fe4f:9f91] QP[1] QKEY[1280243318] (length 64) libfabric:4137:1704470463::efa:av:efa_av_insert_one():638 Inserting address GID[fe80::41c:4ff:fe4f:9f91] QP[1] QKEY[1280243318] to AV .... libfabric:4137:1704470463::efa:av:ofi_av_insert_addr():291 inserting addr: (null) libfabric:4117:1704470463::efa:av:efa_av_insert_one():638 Inserting address GID[fe80::408:67ff:fe14:3559] QP[1] QKEY[1806222746] to AV .... libfabric:4117:1704470463::efa:av:ofi_av_insert_addr():291 inserting addr: (null) libfabric:4117:1704470463::efa:av:ofi_av_insert_addr():314 fi_addr: 0 libfabric:4137:1704470463::efa:av:ofi_av_insert_addr():314 fi_addr: 0 libfabric:4137:1704470463::efa:av:efa_is_local_peer():64 The peer is local. libfabric:4137:1704470463::shm:av:smr_av_insert():84 fe80::41c:4ff:fe4f:9f91_0001_4c4ef676_03e8 libfabric:4137:1704470463::shm:av:ofi_av_insert_addr():291 inserting addr: (null) libfabric:4137:1704470463::shm:av:ofi_av_insert_addr():314 fi_addr: 0 libfabric:4137:1704470463::shm:av:smr_av_insert():103 fi_addr: 0 libfabric:4137:1704470463::efa:av:efa_conn_rdm_init():356 Successfully inserted fe80::41c:4ff:fe4f:9f91_0001_4c4ef676_03e8 to shm provider's av. efa_fiaddr: 0 shm_fiaddr = 0 libfabric:4137:1704470463::efa:av:efa_av_insert_one():661 Successfully inserted address GID[fe80::41c:4ff:fe4f:9f91] QP[1] QKEY[1280243318] to AV. fi_addr: 0 libfabric:4117:1704470463::efa:av:efa_is_local_peer():64 The peer is local. libfabric:4117:1704470463::shm:av:smr_av_insert():84 fe80::408:67ff:fe14:3559_0001_6ba8c59a_03e8 libfabric:4117:1704470463::shm:av:ofi_av_insert_addr():291 inserting addr: (null) libfabric:4117:1704470463::shm:av:ofi_av_insert_addr():314 fi_addr: 0 libfabric:4117:1704470463::shm:av:smr_av_insert():103 fi_addr: 0 libfabric:4117:1704470463::efa:av:efa_conn_rdm_init():356 Successfully inserted fe80::408:67ff:fe14:3559_0001_6ba8c59a_03e8 to shm provider's av. efa_fiaddr: 0 shm_fiaddr = 0 libfabric:4117:1704470463::efa:av:efa_av_insert_one():661 Successfully inserted address GID[fe80::408:67ff:fe14:3559] QP[1] QKEY[1806222746] to AV. fi_addr: 0 libfabric:4117:1704470463::efa:av:efa_av_insert_one():638 Inserting address GID[fe80::41c:4ff:fe4f:9f91] QP[1] QKEY[1280243318] to AV .... libfabric:4117:1704470463::efa:av:ofi_av_insert_addr():291 inserting addr: (null) libfabric:4117:1704470463::efa:av:ofi_av_insert_addr():314 fi_addr: 1 libfabric:4137:1704470463::efa:av:efa_av_insert_one():638 Inserting address GID[fe80::408:67ff:fe14:3559] QP[1] QKEY[1806222746] to AV .... libfabric:4137:1704470463::efa:av:ofi_av_insert_addr():291 inserting addr: (null) libfabric:4137:1704470463::efa:av:ofi_av_insert_addr():314 fi_addr: 1 libfabric:4137:1704470463::efa:av:efa_av_insert_one():661 Successfully inserted address GID[fe80::408:67ff:fe14:3559] QP[1] QKEY[1806222746] to AV. fi_addr: 1 libfabric:4117:1704470463::efa:av:efa_av_insert_one():661 Successfully inserted address GID[fe80::41c:4ff:fe4f:9f91] QP[1] QKEY[1280243318] to AV. fi_addr: 1 #---------------------------------------------------------------- # Intel(R) MPI Benchmarks 2018, MPI-1 part #---------------------------------------------------------------- # Date : Fri Jan 5 16:01:03 2024 # Machine : x86_64 # System : Linux # Release : 4.18.0-477.10.1.el8_8.x86_64 # Version : #1 SMP Tue May 16 11:38:37 UTC 2023 # MPI Version : 3.1 # MPI Thread Environment: # Calling sequence was: # /home/rocky/mpi-benchmarks/src_c/IMB-MPI1 -msglog 3:28 PingPong # Minimum message length in bytes: 0 # Maximum message length in bytes: 268435456 # # MPI_Datatype : MPI_BYTE # MPI_Datatype for reductions : MPI_FLOAT # MPI_Op : MPI_SUM # # # List of Benchmarks to run: # PingPong libfabric:4137:1704470463::efa:cq:efa_rdm_pke_handle_handshake_recv():108 Received peer host id: i-0339cefba46aa889a libfabric:4137:1704470463::efa:cq:efa_rdm_pke_handle_handshake_recv():112 Received peer EFA device version: 0xefa0 libfabric:4117:1704470463::efa:cq:efa_rdm_pke_handle_handshake_recv():108 Received peer host id: i-0ef5266ddb677e20a libfabric:4117:1704470463::efa:cq:efa_rdm_pke_handle_handshake_recv():112 Received peer EFA device version: 0xefa0 #--------------------------------------------------- # Benchmarking PingPong # #processes = 2 #--------------------------------------------------- #bytes #repetitions t[usec] Mbytes/sec 0 1000 18.29 0.00 8 1000 18.80 0.43 16 1000 18.85 0.85 32 1000 18.83 1.70 64 1000 18.85 3.39 128 1000 18.90 6.77 256 1000 18.97 13.50 512 1000 19.12 26.78 1024 1000 19.54 52.41 2048 1000 20.51 99.84 4096 1000 22.73 180.22 8192 1000 27.62 296.64 16384 1000 29.40 557.26 32768 1000 34.21 957.71 65536 640 41.73 1570.37 131072 320 98.13 1335.75 262144 160 118.58 2210.73 524288 80 209.96 2497.14 1048576 40 394.82 2655.81 2097152 20 764.56 2742.97 4194304 10 1447.77 2897.07 8388608 5 2870.17 2922.68 16777216 2 6053.21 2771.62 33554432 1 12819.13 2617.53 67108864 1 25770.85 2604.06 134217728 1 50815.67 2641.27 268435456 1 101673.59 2640.17 # All processes entering MPI_Finalize libfabric:4137:1704470467::shm:av:smr_av_remove():165 0 libfabric:4137:1704470467::efa:av:efa_conn_release():597 efa_conn released! conn[0x7fa6706fe9f0] GID[fe80::41c:4ff:fe4f:9f91] QP[1] libfabric:4117:1704470467::shm:av:smr_av_remove():165 0 libfabric:4117:1704470467::efa:av:efa_conn_release():597 efa_conn released! conn[0x7fac075b09f0] GID[fe80::408:67ff:fe14:3559] QP[1] libfabric:4137:1704470467::efa:av:efa_conn_release():597 efa_conn released! conn[0x7fa6706ff370] GID[fe80::408:67ff:fe14:3559] QP[1] libfabric:4117:1704470467::efa:av:efa_conn_release():597 efa_conn released! conn[0x7fac075b1370] GID[fe80::41c:4ff:fe4f:9f91] QP[1] libfabric:4137:1704470467::efa:mr:ofi_mr_cache_cleanup():471 MR cache stats: searches 6904, deletes 6904, hits 6887 notify 48 libfabric:4117:1704470467::efa:mr:ofi_mr_cache_cleanup():471 MR cache stats: searches 6904, deletes 6904, hits 6887 notify 48 libfabric:4137:1704470467::efa:mr:ofi_mr_cache_cleanup():471 MR cache stats: searches 0, deletes 0, hits 0 notify 60 libfabric:4117:1704470467::efa:mr:ofi_mr_cache_cleanup():471 MR cache stats: searches 0, deletes 0, hits 0 notify 60