Skip to content

Software Counters #14

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
davideberius wants to merge 8 commits into from
Closed

Software Counters #14

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
a3da3e4
Added the software events driver code along with several new counters…
Aug 30, 2016
c402bb0
Added PAPI sde driver code to ensure there are existing weak symbols …
Jun 28, 2017
b7b23a4
Temporarily removed MPI_T helper functions from being compiled since …
Jul 7, 2017
bb84244
Moved the software counters driver code up to the ompi level.
Jul 11, 2017
5f668c5
Added a new USER_OR_MPI macro for determining which counter the bytes…
Aug 23, 2017
daaf7c1
Switched the timer units over to cycles to avoid division operations.…
Oct 12, 2017
e9b8c0c
Temporarily disabled the PAPI sde component for portability.
Oct 20, 2017
c44f915
Added a --with-spc configure option to enable SPCs in the build. Add…
Dec 6, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 18 additions & 0 deletions configure.ac
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -286,6 +286,24 @@ AS_IF([test "$enable_oshmem" != "no"], [project_oshmem_amc=true], [project_oshme
m4_ifndef([project_oshmem], [project_oshmem_amc=false])
AM_CONDITIONAL([PROJECT_OSHMEM], [test "$project_oshmem_amc" = "true"])

# Enable/Disable Software-Based Performance Counters Capability
AC_ARG_ENABLE(spc,
AC_HELP_STRING([--enable-spc],
[Enable software-based performance counters capability (default: disabled)]))
if test "$enable_spc" = "yes"; then
AC_MSG_RESULT([yes])
SOFTWARE_EVENTS_ENABLE=1
else
AC_MSG_RESULT([no])
SOFTWARE_EVENTS_ENABLE=0
fi
AC_DEFINE_UNQUOTED([SOFTWARE_EVENTS_ENABLE],
[$SOFTWARE_EVENTS_ENABLE],
[If the software-based performance counters capability should be enabled.])
AM_CONDITIONAL(SOFTWARE_EVENTS_ENABLE, test "$SOFTWARE_EVENTS_ENABLE" = "1")

AS_IF([test "$enable_spc" != "no"], [project_spc_amc=true], [project_spc_amc=false])

if test "$enable_binaries" = "no" && test "$enable_dist" = "yes"; then
AC_MSG_WARN([--disable-binaries is incompatible with --enable dist])
AC_MSG_ERROR([Cannot continue])
Expand Down
20 changes: 12 additions & 8 deletions ompi/mca/pml/ob1/pml_ob1.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
#include "opal_stdint.h"
#include "opal/mca/btl/btl.h"
#include "opal/mca/btl/base/base.h"
#include "ompi/runtime/ompi_software_events.h"

#include "ompi/mca/pml/pml.h"
#include "ompi/mca/pml/base/base.h"
Expand Down Expand Up @@ -195,6 +196,7 @@ int mca_pml_ob1_add_comm(ompi_communicator_t* comm)
mca_pml_ob1_recv_frag_t *frag, *next_frag;
mca_pml_ob1_comm_proc_t* pml_proc;
mca_pml_ob1_match_hdr_t* hdr;
opal_timer_t usecs = 0;

if (NULL == pml_comm) {
return OMPI_ERR_OUT_OF_RESOURCE;
Expand Down Expand Up @@ -264,15 +266,17 @@ int mca_pml_ob1_add_comm(ompi_communicator_t* comm)
* situation as the cant_match is only checked when a new fragment is received from
* the network.
*/
SW_EVENT_TIMER_START(OMPI_OOS_MATCH_TIME, &usecs);
OPAL_LIST_FOREACH(frag, &pml_proc->frags_cant_match, mca_pml_ob1_recv_frag_t) {
hdr = &frag->hdr.hdr_match;
/* If the message has the next expected seq from that proc... */
if(hdr->hdr_seq != pml_proc->expected_sequence)
continue;

opal_list_remove_item(&pml_proc->frags_cant_match, (opal_list_item_t*)frag);
goto add_fragment_to_unexpected;
}
hdr = &frag->hdr.hdr_match;
/* If the message has the next expected seq from that proc... */
if(hdr->hdr_seq != pml_proc->expected_sequence)
continue;

opal_list_remove_item(&pml_proc->frags_cant_match, (opal_list_item_t*)frag);
goto add_fragment_to_unexpected;
}
SW_EVENT_TIMER_STOP(OMPI_OOS_MATCH_TIME, &usecs);
} else {
opal_list_append( &pml_proc->frags_cant_match, (opal_list_item_t*)frag );
}
Expand Down
11 changes: 11 additions & 0 deletions ompi/mca/pml/ob1/pml_ob1_isend.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
#include "pml_ob1_sendreq.h"
#include "pml_ob1_recvreq.h"
#include "ompi/peruse/peruse-internal.h"
#include "ompi/runtime/ompi_software_events.h"

/**
* Single usage request. As we allow recursive calls (as an
Expand Down Expand Up @@ -119,6 +120,16 @@ static inline int mca_pml_ob1_send_inline (const void *buf, size_t count,
rc = mca_bml_base_sendi (bml_btl, &convertor, &match, OMPI_PML_OB1_MATCH_HDR_LEN,
size, MCA_BTL_NO_ORDER, MCA_BTL_DES_FLAGS_PRIORITY | MCA_BTL_DES_FLAGS_BTL_OWNERSHIP,
MCA_PML_OB1_HDR_TYPE_MATCH, NULL);

if(OPAL_LIKELY(rc == OPAL_SUCCESS)){
if(tag >= 0){
SW_EVENT_RECORD(OMPI_BYTES_SENT_USER, size);
}
else{
SW_EVENT_RECORD(OMPI_BYTES_SENT_MPI, size);
}
}

if (count > 0) {
opal_convertor_cleanup (&convertor);
}
Expand Down
31 changes: 31 additions & 0 deletions ompi/mca/pml/ob1/pml_ob1_recvfrag.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@
#include "ompi/mca/pml/pml.h"
#include "ompi/peruse/peruse-internal.h"
#include "ompi/memchecker.h"
#include "ompi/runtime/ompi_software_events.h"

#include "pml_ob1.h"
#include "pml_ob1_comm.h"
Expand Down Expand Up @@ -231,6 +232,9 @@ void mca_pml_ob1_recv_frag_callback_match(mca_btl_base_module_t* btl,
&iov_count,
&bytes_received );
match->req_bytes_received = bytes_received;

SW_EVENT_USER_OR_MPI(match->req_recv.req_base.req_tag, (long long)bytes_received, OMPI_BYTES_RECEIVED_USER, OMPI_BYTES_RECEIVED_MPI);

/*
* Unpacking finished, make the user buffer unaccessable again.
*/
Expand Down Expand Up @@ -536,6 +540,11 @@ match_one(mca_btl_base_module_t *btl,
mca_pml_ob1_comm_proc_t *proc,
mca_pml_ob1_recv_frag_t* frag)
{
#if SOFTWARE_EVENTS_ENABLE == 1
opal_timer_t usecs = 0;
#endif
SW_EVENT_TIMER_START(OMPI_MATCH_TIME, &usecs);

mca_pml_ob1_recv_request_t *match;
mca_pml_ob1_comm_t *comm = (mca_pml_ob1_comm_t *)comm_ptr->c_pml_comm;

Expand Down Expand Up @@ -573,26 +582,40 @@ match_one(mca_btl_base_module_t *btl,
num_segments);
/* this frag is already processed, so we want to break out
of the loop and not end up back on the unexpected queue. */
SW_EVENT_TIMER_STOP(OMPI_MATCH_TIME, &usecs);

return NULL;
}

PERUSE_TRACE_COMM_EVENT(PERUSE_COMM_MSG_MATCH_POSTED_REQ,
&(match->req_recv.req_base), PERUSE_RECV);
SW_EVENT_TIMER_STOP(OMPI_MATCH_TIME, &usecs);

return match;
}

/* if no match found, place on unexpected queue */
append_frag_to_list(&proc->unexpected_frags, btl, hdr, segments,
num_segments, frag);

SW_EVENT_RECORD(OMPI_UNEXPECTED, 1);

PERUSE_TRACE_MSG_EVENT(PERUSE_COMM_MSG_INSERT_IN_UNEX_Q, comm_ptr,
hdr->hdr_src, hdr->hdr_tag, PERUSE_RECV);
SW_EVENT_TIMER_STOP(OMPI_MATCH_TIME, &usecs);

return NULL;
} while(true);
}

static mca_pml_ob1_recv_frag_t* check_cantmatch_for_match(mca_pml_ob1_comm_proc_t *proc)
{
mca_pml_ob1_recv_frag_t *frag;
#if SOFTWARE_EVENTS_ENABLE == 1
opal_timer_t usecs = 0;
#endif

SW_EVENT_TIMER_START(OMPI_OOS_MATCH_TIME, &usecs);

/* search the list for a fragment from the send with sequence
* number next_msg_seq_expected
Expand All @@ -609,9 +632,14 @@ static mca_pml_ob1_recv_frag_t* check_cantmatch_for_match(mca_pml_ob1_comm_proc_
continue;

opal_list_remove_item(&proc->frags_cant_match, (opal_list_item_t*)frag);

SW_EVENT_TIMER_STOP(OMPI_OOS_MATCH_TIME, &usecs);

return frag;
}

SW_EVENT_TIMER_STOP(OMPI_OOS_MATCH_TIME, &usecs);

return NULL;
}

Expand Down Expand Up @@ -776,6 +804,9 @@ static int mca_pml_ob1_recv_frag_match( mca_btl_base_module_t *btl,
*/
append_frag_to_list(&proc->frags_cant_match, btl, hdr, segments,
num_segments, NULL);

SW_EVENT_RECORD(OMPI_OUT_OF_SEQUENCE, 1);

OB1_MATCHING_UNLOCK(&comm->matching_lock);
return OMPI_SUCCESS;
}
Expand Down
22 changes: 22 additions & 0 deletions ompi/mca/pml/ob1/pml_ob1_recvreq.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@

#include "opal/mca/mpool/mpool.h"
#include "opal/util/arch.h"
#include "ompi/runtime/ompi_software_events.h"
#include "ompi/mca/pml/pml.h"
#include "ompi/mca/bml/bml.h"
#include "pml_ob1_comm.h"
Expand Down Expand Up @@ -198,7 +199,10 @@ static void mca_pml_ob1_put_completion (mca_pml_ob1_rdma_frag_t *frag, int64_t r
assert ((uint64_t) rdma_size == frag->rdma_length);

/* check completion status */

OPAL_THREAD_ADD_SIZE_T(&recvreq->req_bytes_received, rdma_size);
SW_EVENT_USER_OR_MPI(recvreq->req_recv.req_base.req_tag, (long long)rdma_size, OMPI_BYTES_RECEIVED_USER, OMPI_BYTES_RECEIVED_MPI);

if (recv_request_pml_complete_check(recvreq) == false &&
recvreq->req_rdma_offset < recvreq->req_send_offset) {
/* schedule additional rdma operations */
Expand Down Expand Up @@ -242,6 +246,8 @@ int mca_pml_ob1_recv_request_ack_send_btl(
des->des_cbfunc = mca_pml_ob1_recv_ctl_completion;

rc = mca_bml_base_send(bml_btl, des, MCA_PML_OB1_HDR_TYPE_ACK);
SW_EVENT_RECORD(OMPI_BYTES_RECEIVED_MPI, (long long)size);

if( OPAL_LIKELY( rc >= 0 ) ) {
return OMPI_SUCCESS;
}
Expand Down Expand Up @@ -374,6 +380,7 @@ static void mca_pml_ob1_rget_completion (mca_btl_base_module_t* btl, struct mca_
} else {
/* is receive request complete */
OPAL_THREAD_ADD_SIZE_T(&recvreq->req_bytes_received, frag->rdma_length);
SW_EVENT_USER_OR_MPI(recvreq->req_recv.req_base.req_tag, (long long)frag->rdma_length, OMPI_BYTES_RECEIVED_USER, OMPI_BYTES_RECEIVED_MPI);
/* TODO: re-add order */
mca_pml_ob1_send_fin (recvreq->req_recv.req_base.req_proc,
bml_btl, frag->rdma_hdr.hdr_rget.hdr_frag,
Expand Down Expand Up @@ -429,6 +436,10 @@ static int mca_pml_ob1_recv_request_put_frag (mca_pml_ob1_rdma_frag_t *frag)

/* send rdma request to peer */
rc = mca_bml_base_send (bml_btl, ctl, MCA_PML_OB1_HDR_TYPE_PUT);

/* Increment counter for bytes_put even though they probably haven't all been received yet */
SW_EVENT_RECORD(OMPI_BYTES_PUT, frag->rdma_length);

if (OPAL_UNLIKELY(rc < 0)) {
mca_bml_base_free (bml_btl, ctl);
return rc;
Expand Down Expand Up @@ -470,6 +481,10 @@ int mca_pml_ob1_recv_request_get_frag (mca_pml_ob1_rdma_frag_t *frag)
rc = mca_bml_base_get (bml_btl, frag->local_address, frag->remote_address, local_handle,
(mca_btl_base_registration_handle_t *) frag->remote_handle, frag->rdma_length,
0, MCA_BTL_NO_ORDER, mca_pml_ob1_rget_completion, frag);

/* Increment counter for bytes_get even though they probably haven't all been received yet */
SW_EVENT_RECORD(OMPI_BYTES_GET, frag->rdma_length);

if( OPAL_UNLIKELY(OMPI_SUCCESS != rc) ) {
return mca_pml_ob1_recv_request_get_frag_failed (frag, OMPI_ERR_OUT_OF_RESOURCE);
}
Expand Down Expand Up @@ -525,6 +540,8 @@ void mca_pml_ob1_recv_request_progress_frag( mca_pml_ob1_recv_request_t* recvreq
);

OPAL_THREAD_ADD_SIZE_T(&recvreq->req_bytes_received, bytes_received);
SW_EVENT_USER_OR_MPI(recvreq->req_recv.req_base.req_tag, (long long)bytes_received, OMPI_BYTES_RECEIVED_USER, OMPI_BYTES_RECEIVED_MPI);

/* check completion status */
if(recv_request_pml_complete_check(recvreq) == false &&
recvreq->req_rdma_offset < recvreq->req_send_offset) {
Expand Down Expand Up @@ -602,6 +619,7 @@ void mca_pml_ob1_recv_request_frag_copy_finished( mca_btl_base_module_t* btl,
des->des_cbfunc(NULL, NULL, des, 0);

OPAL_THREAD_ADD_SIZE_T(&recvreq->req_bytes_received, bytes_received);
SW_EVENT_USER_OR_MPI(recvreq->req_recv.req_base.req_tag, (long long)bytes_received, OMPI_BYTES_RECEIVED_USER, OMPI_BYTES_RECEIVED_MPI);

/* check completion status */
if(recv_request_pml_complete_check(recvreq) == false &&
Expand Down Expand Up @@ -816,6 +834,7 @@ void mca_pml_ob1_recv_request_progress_rndv( mca_pml_ob1_recv_request_t* recvreq
recvreq->req_recv.req_base.req_datatype);
);
OPAL_THREAD_ADD_SIZE_T(&recvreq->req_bytes_received, bytes_received);
SW_EVENT_USER_OR_MPI(recvreq->req_recv.req_base.req_tag, (long long)bytes_received, OMPI_BYTES_RECEIVED_USER, OMPI_BYTES_RECEIVED_MPI);
}
/* check completion status */
if(recv_request_pml_complete_check(recvreq) == false &&
Expand Down Expand Up @@ -886,6 +905,9 @@ void mca_pml_ob1_recv_request_progress_match( mca_pml_ob1_recv_request_t* recvre
* for this request.
*/
recvreq->req_bytes_received += bytes_received;

SW_EVENT_USER_OR_MPI(recvreq->req_recv.req_base.req_tag, (long long)bytes_received, OMPI_BYTES_RECEIVED_USER, OMPI_BYTES_RECEIVED_MPI);

recv_request_pml_complete(recvreq);
}

Expand Down
Loading