From 23486aee585b145817d385c3082cb9b236010782 Mon Sep 17 00:00:00 2001 From: XuechunHou Date: Thu, 23 Jan 2020 16:40:40 -0700 Subject: [PATCH] added if clause for persistent logging, retrieved clientUID info fixed namespace error retrieved method signiture a new way of retrieving method full name changed to an easier way to retrieve compiled method signature attempted to add -XX:persistentLoggingDatabasePort flag, hit segmentation fault.. added if clause for persistent logging, retrieved clientUID info fixed namespace error retrieved method signiture Attempt to add logger to openj9. make Changes and stuff Add conditionals to check for mongo and cassandra support -XX flags needed for persistent logging completed Fix an issue with my previous merge IMplement environment variable ifdef wrapper thing port initialization on different DBMS added if clause for persistent logging, retrieved clientUID info a new way of retrieving method full name changed to an easier way to retrieve compiled method signature attempted to add -XX:persistentLoggingDatabasePort flag, hit segmentation fault.. retrieved method signiture Attempt to add logger to openj9. Add conditionals to check for mongo and cassandra support -XX flags needed for persistent logging completed IMplement environment variable ifdef wrapper thing add general persistent logging support flag for checking if persistent logging is enabled regardless of dbms fix more merge conflicts I forgot Add more env variable processing Add fix that allows you to enable the cassandra logger debug TR_PersistLogging flag Bring in changes by Ida for persistentLogger spec, dynamically load mongoc library addressed code review feedback addressed code review feedback addressed code review feedback added if clause for persistent logging, retrieved clientUID info fixed namespace error retrieved method signiture a new way of retrieving method full name changed to an easier way to retrieve compiled method signature attempted to add -XX:persistentLoggingDatabasePort flag, hit segmentation fault.. added if clause for persistent logging, retrieved clientUID info fixed namespace error retrieved method signiture Attempt to add logger to openj9. make Changes and stuff Add conditionals to check for mongo and cassandra support -XX flags needed for persistent logging completed Fix an issue with my previous merge IMplement environment variable ifdef wrapper thing port initialization on different DBMS added if clause for persistent logging, retrieved clientUID info a new way of retrieving method full name changed to an easier way to retrieve compiled method signature attempted to add -XX:persistentLoggingDatabasePort flag, hit segmentation fault.. retrieved method signiture Attempt to add logger to openj9. Add conditionals to check for mongo and cassandra support -XX flags needed for persistent logging completed IMplement environment variable ifdef wrapper thing add general persistent logging support flag for checking if persistent logging is enabled regardless of dbms fix more merge conflicts I forgot Add more env variable processing Add fix that allows you to enable the cassandra logger debug TR_PersistLogging flag Bring in changes by Ida for persistentLogger spec, dynamically load mongoc library addressed code review feedback addressed code review feedback addressed code review feedback Begin conversion of mongo to c strings and reformat to code conventions re-enable flags Reformat mongologger addressed code review feedback addressed code review feedback addressed code review feedback Address Feedback. Convert to C Strings. Cleanup Mongoc Finish making c style changes Re-add client UID setter and getter in J9PersistentInfo.cpp Dynamic Load Cassandra Fist address freedback stuff made error messages consistent Convert sprintf to snprintf Convert sprintf to snprintf Remove final debug print --- runtime/compiler/CMakeLists.txt | 12 +- runtime/compiler/build/files/common.mk | 10 + runtime/compiler/build/toolcfg/gnu/common.mk | 14 + .../compiler/control/BasePersistentLogger.hpp | 38 +++ runtime/compiler/control/CMakeLists.txt | 12 + runtime/compiler/control/CassandraLogger.cpp | 229 ++++++++++++++++ runtime/compiler/control/CassandraLogger.hpp | 28 ++ runtime/compiler/control/HookedByTheJit.cpp | 10 +- runtime/compiler/control/J9Options.cpp | 60 ++++- .../control/JITServerCompilationThread.cpp | 46 ++++ runtime/compiler/control/LoadDBLibs.cpp | 248 ++++++++++++++++++ runtime/compiler/control/LoadDBLibs.hpp | 233 ++++++++++++++++ runtime/compiler/control/MongoLogger.cpp | 161 ++++++++++++ runtime/compiler/control/MongoLogger.hpp | 36 +++ runtime/compiler/control/rossa.cpp | 21 ++ runtime/compiler/env/J9PersistentInfo.hpp | 34 +++ runtime/compiler/env/j9methodServer.cpp | 20 +- runtime/compiler/env/j9methodServer.hpp | 2 +- 18 files changed, 1197 insertions(+), 17 deletions(-) create mode 100644 runtime/compiler/control/BasePersistentLogger.hpp create mode 100644 runtime/compiler/control/CassandraLogger.cpp create mode 100644 runtime/compiler/control/CassandraLogger.hpp create mode 100644 runtime/compiler/control/LoadDBLibs.cpp create mode 100644 runtime/compiler/control/LoadDBLibs.hpp create mode 100644 runtime/compiler/control/MongoLogger.cpp create mode 100644 runtime/compiler/control/MongoLogger.hpp diff --git a/runtime/compiler/CMakeLists.txt b/runtime/compiler/CMakeLists.txt index b690cb7412d..7ee66009bcd 100644 --- a/runtime/compiler/CMakeLists.txt +++ b/runtime/compiler/CMakeLists.txt @@ -80,7 +80,7 @@ add_custom_command( ) add_custom_target(j9jit_tracegen DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/env/ut_j9jit.h) -# J9VM_OPT_JITSERVER +# J9VM_OPT_JITSERVER and protobuf and Persistent Loggers if(J9VM_OPT_JITSERVER) message(STATUS "JITServer is supported") @@ -99,6 +99,16 @@ if(J9VM_OPT_JITSERVER) include(FindOpenSSL) find_package(OpenSSL REQUIRED) include_directories(${OPENSSL_INCLUDE_DIR}) + + #Check for environment variable enabling the persistent logger + if(${PERSISTENT_LOGGER} STREQUAL "MONGODB") + add_definitions(-DMONGO_LOGGER) + add_definitions(-DPERSISTENT_LOGGING_SUPPORT) + endif() + if(${PERSISTENT_LOGGER} STREQUAL "CASSANDRA") + add_definitions(-DCASSANDRA_LOGGER) + add_definitions(-DPERSISTENT_LOGGING_SUPPORT) + endif() endif() #TODO We should get rid of this, but its still required by the compiler_support module in omr diff --git a/runtime/compiler/build/files/common.mk b/runtime/compiler/build/files/common.mk index b11fc2df578..8ecce35ed14 100644 --- a/runtime/compiler/build/files/common.mk +++ b/runtime/compiler/build/files/common.mk @@ -382,6 +382,16 @@ JIT_PRODUCT_SOURCE_FILES+=\ omr/compiler/runtime/OMRRuntimeAssumptions.cpp ifneq ($(J9VM_OPT_JITSERVER),) +ifeq ($(PERSISTENT_LOGGER),MONGODB) + JIT_PRODUCT_SOURCE_FILES+=\ + compiler/control/MongoLogger.cpp \ + compiler/control/LoadDBLibs.cpp + endif +ifeq ($(PERSISTENT_LOGGER),CASSANDRA) + JIT_PRODUCT_SOURCE_FILES+=\ + compiler/control/CassandraLogger.cpp \ + compiler/control/LoadDBLibs.cpp +endif JIT_PRODUCT_SOURCE_FILES+=\ compiler/control/JITClientCompilationThread.cpp \ compiler/control/JITServerCompilationThread.cpp \ diff --git a/runtime/compiler/build/toolcfg/gnu/common.mk b/runtime/compiler/build/toolcfg/gnu/common.mk index 38c1763e06d..fad81af25ce 100644 --- a/runtime/compiler/build/toolcfg/gnu/common.mk +++ b/runtime/compiler/build/toolcfg/gnu/common.mk @@ -534,4 +534,18 @@ ifneq ($(J9VM_OPT_JITSERVER),) C_INCLUDES+=$(OPENSSL_DIR) CXX_INCLUDES+=$(OPENSSL_DIR) endif + + ifeq ($(PERSISTENT_LOGGER),CASSANDRA) + SOLINK_SLINK+=cassandra + CXX_DEFINES+=CASSANDRA_LOGGER + CXX_DEFINES+=PERSISTENT_LOGGING_SUPPORT + endif + ifeq ($(PERSISTENT_LOGGER),MONGODB) + SOLINK_SLINK+=bsoncxx + SOLINK_SLINK+=mongocxx + CXX_DEFINES+=MONGO_LOGGER + CXX_DEFINES+=PERSISTENT_LOGGING_SUPPORT + CXX_INCLUDES+=/usr/include/mongocxx/v_noabi + CXX_INCLUDES+=/usr/include/bsoncxx/v_noabi + endif endif # J9VM_OPT_JITSERVER diff --git a/runtime/compiler/control/BasePersistentLogger.hpp b/runtime/compiler/control/BasePersistentLogger.hpp new file mode 100644 index 00000000000..d18cc955ca8 --- /dev/null +++ b/runtime/compiler/control/BasePersistentLogger.hpp @@ -0,0 +1,38 @@ +#ifndef JITSERVERLOGGER_BASEPERSISTENTLOGGER_H +#define JITSERVERLOGGER_BASEPERSISTENTLOGGER_H +#include + +class BasePersistentLogger + { + protected: + const char * _databaseIP; + uint32_t _databasePort; + const char* _databaseUsername; + const char* _databasePassword; + const char* _databaseName; + + public: + virtual bool connect() = 0; + virtual void disconnect() = 0; + + BasePersistentLogger(const char * databaseIP, uint32_t databasePort, const char * databaseName) + { + _databaseIP = databaseIP; + _databasePort = databasePort; + _databaseName = databaseName; + _databaseUsername = ""; + _databasePassword = ""; + } + BasePersistentLogger(const char *databaseIP, uint32_t databasePort, const char *databaseName, + const char *databaseUsername, const char *databasePassword) + { + _databaseIP = databaseIP; + _databasePort = databasePort; + _databaseName = databaseName; + _databaseUsername = databaseUsername; + _databasePassword = databasePassword; + } + virtual bool logMethod(const char* method, uint64_t clientID, const char *logContent) = 0; + }; + +#endif //JITSERVERLOGGER_BASEPERSISTENTLOGGER_H diff --git a/runtime/compiler/control/CMakeLists.txt b/runtime/compiler/control/CMakeLists.txt index 9f7632920d6..e67497fd3fc 100644 --- a/runtime/compiler/control/CMakeLists.txt +++ b/runtime/compiler/control/CMakeLists.txt @@ -32,6 +32,18 @@ j9jit_files( ) if(J9VM_OPT_JITSERVER) + if(${PERSISTENT_LOGGER} STREQUAL "MONGODB") + j9jit_files( + control/MongoLogger.cpp + control/LoadDBLibs.cpp + ) + endif() + if(${PERSISTENT_LOGGER} STREQUAL "CASSANDRA") + j9jit_files( + control/CassandraLogger.cpp + control/LoadDBLibs.cpp + ) + endif() j9jit_files( control/JITClientCompilationThread.cpp control/JITServerCompilationThread.cpp diff --git a/runtime/compiler/control/CassandraLogger.cpp b/runtime/compiler/control/CassandraLogger.cpp new file mode 100644 index 00000000000..f505ac8a361 --- /dev/null +++ b/runtime/compiler/control/CassandraLogger.cpp @@ -0,0 +1,229 @@ +#include +#include + +#include "CassandraLogger.hpp" +#include "LoadDBLibs.hpp" +CassandraLogger::CassandraLogger(const char *databaseIP, +uint32_t databasePort, +const char *databaseName): BasePersistentLogger(databaseIP, databasePort, databaseName) + { + _session = NULL; + _connectFuture = NULL; + _cluster = NULL; + } + +CassandraLogger::CassandraLogger(const char *databaseIP, uint32_t databasePort, + const char *databaseName, const char *databaseUsername, const char *databasePassword) + : BasePersistentLogger(databaseIP, databasePort, databaseName, databaseUsername, databasePassword) + { + _session = NULL; + _connectFuture = NULL; + _cluster = NULL; + } + +bool CassandraLogger::createKeySpace() + { + char queryString[256]; + snprintf(queryString, 256, "CREATE KEYSPACE IF NOT EXISTS %s WITH REPLICATION = {'class':'SimpleStrategy','replication_factor':1};", _databaseName); + OCassStatement* statement = Ocass_statement_new(queryString, 0); + + OCassFuture* queryFuture = Ocass_session_execute(_session, statement); + Ocass_statement_free(statement); + if (Ocass_future_error_code(queryFuture) != 0) + { + /* Display connection error message */ + const char* message; + size_t messageLength; + Ocass_future_error_message(queryFuture, &message, &messageLength); + fprintf(stderr, "PersistentLogging - Cassandra Database Keyspace Creation Error: '%.*s'\n", (int)messageLength, message); + + Ocass_future_free(queryFuture); + return false; + } + + Ocass_future_free(queryFuture); + return true; + +} +bool CassandraLogger::createTable(const char *tableName) + { + char queryString[256]; + snprintf(queryString, 256, "CREATE TABLE IF NOT EXISTS %s.%s (clientID text, methodName text, logContent text, insertionDate date,insertionTime time, primary key (clientID, methodName, insertionDate, insertionTime));", _databaseName, tableName); + OCassStatement* statement = Ocass_statement_new(queryString, 0); + OCassFuture* queryFuture = Ocass_session_execute(_session, statement); + Ocass_statement_free(statement); + if (Ocass_future_error_code(queryFuture) != 0) + { + /* Display connection error message */ + const char* message; + size_t messageLength; + Ocass_future_error_message(queryFuture, &message, &messageLength); + fprintf(stderr, "Persistent Logging - Cassandra Database Table Creation Error: '%.*s'\n", (int)messageLength, message); + + Ocass_future_free(queryFuture); + return false; + } + + Ocass_future_free(queryFuture); + return true; + } +bool CassandraLogger::connect() + { + /* Setup and connect to cluster */ + _cluster = Ocass_cluster_new(); + _session = Ocass_session_new(); + /*authenticate using databaseUsername and databasePassword*/ + Ocass_cluster_set_credentials(_cluster, _databaseUsername, _databasePassword); + /*Set protocol version */ + int rc_set_protocol = Ocass_cluster_set_protocol_version(_cluster, 4); + if (rc_set_protocol != 0) + { + fprintf(stderr, "Persistent Logging - Cassandra Database Error: %s\n",Ocass_error_desc(rc_set_protocol)); + Ocass_session_free(_session); + Ocass_cluster_free(_cluster); + return false; + + } + + /* Add contact points */ + int rc_set_ip = Ocass_cluster_set_contact_points(_cluster, _databaseIP); + if (rc_set_ip != 0) + { + fprintf(stderr, "Persistent Logging - Cassandra Database Error: %s\n",Ocass_error_desc(rc_set_ip)); + Ocass_session_free(_session); + Ocass_cluster_free(_cluster); + return false; + + } + + /*Set port number*/ + int rc_set_port = Ocass_cluster_set_port(_cluster, _databasePort); + if (rc_set_port != 0) + { + fprintf(stderr, "Persistent Logging - Cassandra Database Error: %s\n",Ocass_error_desc(rc_set_port)); + Ocass_session_free(_session); + Ocass_cluster_free(_cluster); + + return false; + + } + + /* Provide the cluster object as configuration to connect the session */ + _connectFuture = Ocass_session_connect(_session, _cluster); + + if (Ocass_future_error_code(_connectFuture) != 0) + { + /* Display connection error message */ + const char* message; + size_t messageLength; + Ocass_future_error_message(_connectFuture, &message, &messageLength); + fprintf(stderr, "Persistent Logging - Cassandra Database Connection Error: '%.*s'\n", (int)messageLength, message); + Ocass_session_free(_session); + Ocass_cluster_free(_cluster); + Ocass_future_free(_connectFuture); + return false; + } + return true; + + } + + +bool CassandraLogger::logMethod(const char *method, uint64_t clientID, const char *logContent) + { + + // create table space and table first + if (!createKeySpace()) return false; + const char* tableName = "logs"; + if (!createTable(tableName)) return false; + char queryString[256]; + + snprintf(queryString, 256, "INSERT INTO %s.%s (clientID, methodName, logContent, insertionDate, insertionTime) VALUES (?, ?, ?, ?, ?)", _databaseName,tableName); + OCassStatement* statement + = Ocass_statement_new(queryString, 5); + + /* Bind the values using the indices of the bind variables */ + char strClientID[64]; + snprintf(strClientID, 64, "%lu", clientID); + int rc_set_bind_pk = Ocass_statement_bind_string(statement, 0, strClientID); + if (rc_set_bind_pk != 0) + { + fprintf(stderr, "Persistent Logging - Cassandra Database Error: %s\n",Ocass_error_desc(rc_set_bind_pk)); + Ocass_statement_free(statement); + return false; + } + + int rc_set_bind_method = Ocass_statement_bind_string(statement, 1, method); + if (rc_set_bind_pk != 0) + { + fprintf(stderr, "Persistent Logging - Cassandra Database Error: %s\n",Ocass_error_desc(rc_set_bind_method)); + Ocass_statement_free(statement); + return false; + } + int rc_set_bind_log_content = Ocass_statement_bind_string(statement, 2, logContent); + + if (rc_set_bind_log_content != 0) + { + fprintf(stderr, "Persistent Logging - Cassandra Database Error: %s\n",Ocass_error_desc(rc_set_bind_log_content)); + Ocass_statement_free(statement); + return false; + } + + time_t now = time(NULL); /* Time in seconds from Epoch */ + /* Converts the time since the Epoch in seconds to the 'date' type */ + Ocass_uint32_t year_month_day_of_insertion = Ocass_date_from_epoch(now); + /* Converts the time since the Epoch in seconds to the 'time' type */ + Ocass_int64_t time_of_insertion = Ocass_time_from_epoch(now); + + /* 'date' uses an unsigned 32-bit integer */ + int rc_set_bind_insertion_date = Ocass_statement_bind_uint32(statement, 3, year_month_day_of_insertion); + + if (rc_set_bind_insertion_date != 0) + { + fprintf(stderr, "Persistent Logging - Cassandra Database Error: %s\n", Ocass_error_desc(rc_set_bind_insertion_date)); + Ocass_statement_free(statement); + return false; + } + /* 'time' uses a signed 64-bit integer */ + int rc_set_bind_insertion_time = Ocass_statement_bind_int64(statement, 4, time_of_insertion); + if (rc_set_bind_insertion_time != 0) + { + fprintf(stderr ,"Persistent Logging - Cassandra Database Error: %s\n",Ocass_error_desc(rc_set_bind_insertion_time)); + Ocass_statement_free(statement); + return false; + } + + OCassFuture* queryFuture = Ocass_session_execute(_session, statement); + + /* Statement objects can be freed immediately after being executed */ + Ocass_statement_free(statement); + if (Ocass_future_error_code(queryFuture) != 0) + { + /* Display connection error message */ + const char* message; + size_t messageLength; + Ocass_future_error_message(queryFuture, &message, &messageLength); + fprintf(stderr, "Persistent Logging - Cassandra Database Query Execution Error: '%.*s'\n", (int)messageLength, message); + Ocass_future_free(queryFuture); + return false; + } + + Ocass_future_free(queryFuture); + return true; + } + + +void CassandraLogger::disconnect() + { + Ocass_future_free(_connectFuture); + Ocass_session_free(_session); + Ocass_cluster_free(_cluster); + } + + + + + + + + + diff --git a/runtime/compiler/control/CassandraLogger.hpp b/runtime/compiler/control/CassandraLogger.hpp new file mode 100644 index 00000000000..358028eb74f --- /dev/null +++ b/runtime/compiler/control/CassandraLogger.hpp @@ -0,0 +1,28 @@ +#ifndef CASSANDRALOGGER_H +#define CASSANDRALOGGER_H + +#include "LoadDBLibs.hpp" +#include "BasePersistentLogger.hpp" +class CassandraLogger : public BasePersistentLogger + { + + private: + OCassCluster* _cluster; + OCassSession* _session; + OCassFuture* _connectFuture; + bool createKeySpace(); + bool createTable(const char *tableName); + + + public: + + bool connect() override; + void disconnect() override; + CassandraLogger(const char *databaseIP, uint32_t databasePort, const char *databaseName, + const char *databaseUsername, const char *databasePassword); + CassandraLogger(const char *databaseIP, uint32_t databasePort, const char *databaseName); + bool logMethod(const char *method, uint64_t clientID, const char *logContent) override; + + }; + +#endif // CASSANDRALOGGER_H \ No newline at end of file diff --git a/runtime/compiler/control/HookedByTheJit.cpp b/runtime/compiler/control/HookedByTheJit.cpp index a90aa87d370..4111ac97642 100644 --- a/runtime/compiler/control/HookedByTheJit.cpp +++ b/runtime/compiler/control/HookedByTheJit.cpp @@ -82,7 +82,10 @@ #include "runtime/JITServerIProfiler.hpp" #include "runtime/JITServerStatisticsThread.hpp" #include "runtime/Listener.hpp" -#endif +#if defined(MONGO_LOGGER) +#include "control/LoadDBLibs.hpp" +#endif // defined(MONGO_LOGGER) +#endif // defined(J9VM_OPT_JITSERVER) extern "C" { struct J9JavaVM; @@ -4733,6 +4736,11 @@ void JitShutdown(J9JITConfig * jitConfig) } } +#if defined(J9VM_OPT_JITSERVER) && defined(MONGO_LOGGER) + if(compInfo->getPersistentInfo()->getJITServerPersistentLogging()) + JITServer::cleanupMongoC(); +#endif //J9VM_OPT_JITSERVER && MONGO_LOGGER + TR::Compilation::shutdown(vm); TR::CompilationController::shutdown(); diff --git a/runtime/compiler/control/J9Options.cpp b/runtime/compiler/control/J9Options.cpp index 9300c9ac68b..eb73c497691 100644 --- a/runtime/compiler/control/J9Options.cpp +++ b/runtime/compiler/control/J9Options.cpp @@ -1074,13 +1074,26 @@ static void JITServerParseCommonOptions(J9JavaVM *vm, TR::CompilationInfo *compI const char *xxJITServerSSLKeyOption = "-XX:JITServerSSLKey="; const char *xxJITServerSSLCertOption = "-XX:JITServerSSLCert="; const char *xxJITServerSSLRootCertsOption = "-XX:JITServerSSLRootCerts="; - int32_t xxJITServerPortArgIndex = FIND_ARG_IN_VMARGS(STARTSWITH_MATCH, xxJITServerPortOption, 0); int32_t xxJITServerTimeoutArgIndex = FIND_ARG_IN_VMARGS(STARTSWITH_MATCH, xxJITServerTimeoutOption, 0); int32_t xxJITServerSSLKeyArgIndex = FIND_ARG_IN_VMARGS(STARTSWITH_MATCH, xxJITServerSSLKeyOption, 0); int32_t xxJITServerSSLCertArgIndex = FIND_ARG_IN_VMARGS(STARTSWITH_MATCH, xxJITServerSSLCertOption, 0); int32_t xxJITServerSSLRootCertsArgIndex = FIND_ARG_IN_VMARGS(STARTSWITH_MATCH, xxJITServerSSLRootCertsOption, 0); - +#if defined(MONGO_LOGGER) || defined(CASSANDRA_LOGGER) + const char *xxJITServerPersistentLoggingOption = "-XX:JITServerPersistentLogging="; + const char *xxJITServerPersistentLoggingDatabasePortOption = "-XX:JITServerPersistentLoggingDatabasePort="; + const char *xxJITServerPersistentLoggingDatabaseAddressOption = "-XX:JITServerPersistentLoggingDatabaseAddress="; + const char *xxJITServerPersistentLoggingDatabaseNameOption = "-XX:JITServerPersistentLoggingDatabaseName="; + const char *xxJITServerPersistentLoggingDatabaseUsernameOption = "-XX:JITServerPersistentLoggingDatabaseUsername="; + const char *xxJITServerPersistentLoggingDatabasePasswordOption = "-XX:JITServerPersistentLoggingDatabasePassword="; + int32_t xxJITServerPersistentLoggingArgIndex = FIND_ARG_IN_VMARGS(STARTSWITH_MATCH, xxJITServerPersistentLoggingOption, 0); + int32_t xxJITServerPersistentLoggingDatabasePortArgIndex = FIND_ARG_IN_VMARGS(STARTSWITH_MATCH, xxJITServerPersistentLoggingDatabasePortOption, 0); + int32_t xxJITServerPersistentLoggingDatabaseAddressArgIndex = FIND_ARG_IN_VMARGS(STARTSWITH_MATCH, xxJITServerPersistentLoggingDatabaseAddressOption, 0); + int32_t xxJITServerPersistentLoggingDatabaseNameArgIndex = FIND_ARG_IN_VMARGS(STARTSWITH_MATCH, xxJITServerPersistentLoggingDatabaseNameOption, 0); + int32_t xxJITServerPersistentLoggingDatabaseUsernameArgIndex = FIND_ARG_IN_VMARGS(STARTSWITH_MATCH, xxJITServerPersistentLoggingDatabaseUsernameOption, 0); + int32_t xxJITServerPersistentLoggingDatabasePasswordArgIndex = FIND_ARG_IN_VMARGS(STARTSWITH_MATCH, xxJITServerPersistentLoggingDatabasePasswordOption, 0); +#endif // defined(MONGO_LOGGER) || ... + if (xxJITServerPortArgIndex >= 0) { uint32_t port=0; @@ -1089,6 +1102,49 @@ static void JITServerParseCommonOptions(J9JavaVM *vm, TR::CompilationInfo *compI compInfo->getPersistentInfo()->setJITServerPort(port); } +#if defined(MONGO_LOGGER) || defined(CASSANDRA_LOGGER) + if (xxJITServerPersistentLoggingArgIndex >= 0) + { + bool enable = false; + IDATA ret = GET_INTEGER_VALUE(xxJITServerPersistentLoggingArgIndex, xxJITServerPersistentLoggingOption, enable); + if (ret == OPTION_OK) + compInfo->getPersistentInfo()->setJITServerPersistentLogging(enable); + } + if (xxJITServerPersistentLoggingDatabasePortArgIndex >= 0) + { + uint32_t port=0; + IDATA ret = GET_INTEGER_VALUE(xxJITServerPersistentLoggingDatabasePortArgIndex, xxJITServerPersistentLoggingDatabasePortOption, port); + if (ret == OPTION_OK) + compInfo->getPersistentInfo()->setJITServerPersistentLoggingDatabasePort(port); + } + if (xxJITServerPersistentLoggingDatabaseUsernameArgIndex >= 0) + { + char *username = NULL; + GET_OPTION_VALUE(xxJITServerPersistentLoggingDatabaseUsernameArgIndex, '=', &username); + compInfo->getPersistentInfo()->setJITServerPersistentLoggingDatabaseUsername(username); + } + if (xxJITServerPersistentLoggingDatabaseNameArgIndex >= 0) + { + char *name = NULL; + GET_OPTION_VALUE(xxJITServerPersistentLoggingDatabaseNameArgIndex, '=', &name); + compInfo->getPersistentInfo()->setJITServerPersistentLoggingDatabaseName(name); + } + + if (xxJITServerPersistentLoggingDatabasePasswordArgIndex >= 0) + { + char *password = NULL; + GET_OPTION_VALUE(xxJITServerPersistentLoggingDatabasePasswordArgIndex, '=', &password); + compInfo->getPersistentInfo()->setJITServerPersistentLoggingDatabasePassword(password); + } + + if (xxJITServerPersistentLoggingDatabaseAddressArgIndex >= 0) + { + char *address = NULL; + GET_OPTION_VALUE(xxJITServerPersistentLoggingDatabaseAddressArgIndex, '=', &address); + compInfo->getPersistentInfo()->setJITServerPersistentLoggingDatabaseAddress(address); + } +#endif // defined(MONGO_LOGGER) || ... + if (xxJITServerTimeoutArgIndex >= 0) { uint32_t timeoutMs=0; diff --git a/runtime/compiler/control/JITServerCompilationThread.cpp b/runtime/compiler/control/JITServerCompilationThread.cpp index 3b21855c7c0..954edec882e 100644 --- a/runtime/compiler/control/JITServerCompilationThread.cpp +++ b/runtime/compiler/control/JITServerCompilationThread.cpp @@ -37,6 +37,13 @@ #include "net/ServerStream.hpp" #include "jitprotos.h" #include "vmaccess.h" +#ifdef CASSANDRA_LOGGER +#include "control/CassandraLogger.hpp" +#endif // CASSANDRA_LOGGER +#ifdef MONGO_LOGGER +#include "control/MongoLogger.hpp" +#endif // MONGO_LOGGER + /** * @brief Method executed by JITServer to process the end of a compilation. @@ -83,6 +90,45 @@ outOfProcessCompilationEnd( // Pack log file to send to client std::string logFileStr = TR::Options::packLogFile(comp->getOutFile()); +#if defined(MONGO_LOGGER) || defined(CASSANDRA_LOGGER) + if (compInfoPT->getCompilationInfo()->getPersistentInfo()->getJITServerPersistentLogging() + && comp->getOption(TR_PersistentLogging)) //Check both server and client enable persistent logging. + { + uint64_t clientUID = entry->getClientUID(); + const char* methodSignature = compInfoPT->getCompilation()->signature(); + TR::PersistentInfo* persistentInfo = compInfoPT->getCompilationInfo()->getPersistentInfo(); + uint32_t persistentLoggingDatabasePort = persistentInfo->getJITServerPersistentLoggingDatabasePort(); + const char* persistentLoggingDatabaseAddress = persistentInfo->getJITServerPersistentLoggingDatabaseAddress(); + const char* persistentLoggingDatabaseUsername = persistentInfo->getJITServerPersistentLoggingDatabaseUsername(); + const char* persistentLoggingDatabasePassword = persistentInfo->getJITServerPersistentLoggingDatabasePassword(); + const char* persistentLoggingDatabaseName = persistentInfo->getJITServerPersistentLoggingDatabaseName(); + +#if defined(MONGO_LOGGER) + MongoLogger logger(persistentLoggingDatabaseAddress, + persistentLoggingDatabasePort, + persistentLoggingDatabaseName, + persistentLoggingDatabaseUsername, + persistentLoggingDatabasePassword); +#elif defined(CASSANDRA_LOGGER) + CassandraLogger logger(persistentLoggingDatabaseAddress, + persistentLoggingDatabasePort, + persistentLoggingDatabaseName, + persistentLoggingDatabaseUsername, + persistentLoggingDatabasePassword); +#endif // MONGO_LOGGER elif CASSANDRA_LOGGER + + bool isConnected = logger.connect(); + if (isConnected) + { + if (!logger.logMethod(methodSignature, clientUID, logFileStr.c_str())) + fprintf(stderr, "JITServer: Persistent Logging Error - Database insert failed, skipping persistent logging."); + logger.disconnect(); + } + else + fprintf(stderr, "JITServer: Persistent Logging Error - Database connection failed.\n"); + } +#endif // defined(MONGO_LOGGER) || defined(CASSANDRA_LOGGER) + std::string svmSymbolToIdStr; if (comp->getOption(TR_UseSymbolValidationManager)) { diff --git a/runtime/compiler/control/LoadDBLibs.cpp b/runtime/compiler/control/LoadDBLibs.cpp new file mode 100644 index 00000000000..d9060997f57 --- /dev/null +++ b/runtime/compiler/control/LoadDBLibs.cpp @@ -0,0 +1,248 @@ +// +// Created by cmbuhler on 2020-03-11. +// +#include "LoadDBLibs.hpp" + +#include +#include +#include + +#if defined(MONGO_LOGGER) +/* + * MONGOC and BSON Library functions and strcuts + */ +Obson_new_t *Obson_new = NULL; +Obson_append_utf8_t *Obson_append_utf8 = NULL; +Obson_append_date_time_t *Obson_append_date_time = NULL; +Obson_destroy_t *Obson_destroy = NULL; + +Omongoc_init_t *Omongoc_init = NULL; +Omongoc_cleanup_t *Omongoc_cleanup = NULL; +Omongoc_uri_new_with_error_t *Omongoc_uri_new_with_error = NULL; +Omongoc_client_new_from_uri_t *Omongoc_client_new_from_uri = NULL; +Omongoc_client_set_appname_t *Omongoc_client_set_appname = NULL; +Omongoc_client_get_database_t *Omongoc_client_get_database = NULL; +Omongoc_client_get_collection_t *Omongoc_client_get_collection = NULL; +Omongoc_collection_insert_one_t *Omongoc_collection_insert_one = NULL; +Omongoc_collection_destroy_t *Omongoc_collection_destroy = NULL; +Omongoc_database_destroy_t *Omongoc_database_destroy = NULL; +Omongoc_uri_destroy_t *Omongoc_uri_destroy = NULL; +Omongoc_client_destroy_t *Omongoc_client_destroy = NULL; +#endif // defined(MONGO_LOGGER) + +#if defined(CASSANDRA_LOGGER) +/* + * CASSANDRA functions and structs + */ +Ocass_statement_new_t *Ocass_statement_new = NULL; +Ocass_statement_free_t *Ocass_statement_free = NULL; +Ocass_session_execute_t *Ocass_session_execute = NULL; +Ocass_future_error_code_t *Ocass_future_error_code = NULL; +Ocass_future_error_message_t *Ocass_future_error_message = NULL; +Ocass_future_free_t *Ocass_future_free = NULL; +Ocass_cluster_new_t *Ocass_cluster_new = NULL; +Ocass_session_new_t *Ocass_session_new = NULL; +Ocass_cluster_set_credentials_t *Ocass_cluster_set_credentials = NULL; +Ocass_cluster_set_protocol_version_t *Ocass_cluster_set_protocol_version = NULL; +Ocass_session_free_t *Ocass_session_free = NULL; +Ocass_cluster_free_t *Ocass_cluster_free = NULL; +Ocass_cluster_set_contact_points_t *Ocass_cluster_set_contact_points = NULL; +Ocass_cluster_set_port_t *Ocass_cluster_set_port = NULL; +Ocass_session_connect_t *Ocass_session_connect = NULL; +Ocass_statement_bind_string_t *Ocass_statement_bind_string = NULL; +Ocass_error_desc_t *Ocass_error_desc = NULL; +Ocass_time_from_epoch_t *Ocass_time_from_epoch = NULL; +Ocass_date_from_epoch_t *Ocass_date_from_epoch = NULL; +Ocass_statement_bind_int64_t *Ocass_statement_bind_int64 = NULL; +Ocass_statement_bind_uint32_t *Ocass_statement_bind_uint32 = NULL; +#endif // defined(CASSANDRA_LOGGER) + +namespace JITServer + { +#if defined(MONGO_LOGGER) + void *loadLibmongoc() + { + void *result = NULL; + result = dlopen("libmongoc-1.0.so", RTLD_NOW); + return result; + } + + void *loadLibbson() + { + void *result = NULL; + result = dlopen("libbson-1.0.so", RTLD_NOW); + return result; + } +#endif // defined(MONGO_LOGGER) +#if defined(CASSANDRA_LOGGER) + void *loadLibcassandra() + { + void *result = NULL; + result = dlopen("libcassandra.so", RTLD_NOW); + return result; + } +#endif //defined(CASSANDRA_LOGGER) + + void unloadDBLib(void *handle) + { + (void) dlclose(handle); + } + + void *findDBLibSymbol(void *handle, const char *sym) + { + return dlsym(handle, sym); + } + +#if defined(MONGO_LOGGER) + bool loadLibmongocAndSymbols() + { + void *handle = NULL; + + handle = loadLibmongoc(); + if (!handle) + { + printf("#JITServer: Failed to load libmongoc\n"); + return false; + } + + Omongoc_init = (Omongoc_init_t *) findDBLibSymbol(handle, "mongoc_init"); + Omongoc_cleanup = (Omongoc_cleanup_t *) findDBLibSymbol(handle, "mongoc_cleanup"); + Omongoc_uri_new_with_error = (Omongoc_uri_new_with_error_t *) findDBLibSymbol(handle, + "mongoc_uri_new_with_error"); + Omongoc_client_new_from_uri = (Omongoc_client_new_from_uri_t *) findDBLibSymbol(handle, + "mongoc_client_new_from_uri"); + Omongoc_client_set_appname = (Omongoc_client_set_appname_t *) findDBLibSymbol(handle, + "mongoc_client_set_appname"); + Omongoc_client_get_database = (Omongoc_client_get_database_t *) findDBLibSymbol(handle, + "mongoc_client_get_database"); + Omongoc_client_get_collection = (Omongoc_client_get_collection_t *) findDBLibSymbol(handle, + "mongoc_client_get_collection"); + Omongoc_collection_insert_one = (Omongoc_collection_insert_one_t *) findDBLibSymbol(handle, + "mongoc_collection_insert_one"); + Omongoc_collection_destroy = (Omongoc_collection_destroy_t *) findDBLibSymbol(handle, + "mongoc_collection_destroy"); + Omongoc_database_destroy = (Omongoc_database_destroy_t *) findDBLibSymbol(handle, "mongoc_database_destroy"); + Omongoc_uri_destroy = (Omongoc_uri_destroy_t *) findDBLibSymbol(handle, "mongoc_uri_destroy"); + Omongoc_client_destroy = (Omongoc_client_destroy_t *) findDBLibSymbol(handle, "mongoc_client_destroy"); + + if ( + (Omongoc_init == NULL) || + (Omongoc_cleanup == NULL) || + (Omongoc_uri_new_with_error == NULL) || + (Omongoc_client_new_from_uri == NULL) || + (Omongoc_client_set_appname == NULL) || + (Omongoc_client_get_database == NULL) || + (Omongoc_client_get_collection == NULL) || + (Omongoc_collection_insert_one == NULL) || + (Omongoc_collection_destroy == NULL) || + (Omongoc_database_destroy == NULL) || + (Omongoc_uri_destroy == NULL) || + (Omongoc_client_destroy == NULL) + ) + { + printf("#JITServer: Failed to load all the required Mongoc symbols\n"); + unloadDBLib(handle); + return false; + } + + return true; + } + + bool loadLibbsonAndSymbols() + { + void *handle = NULL; + + handle = loadLibbson(); + if (!handle) + { + printf("#JITServer: Failed to load libbson\n"); + return false; + } + + Obson_new = (Obson_new_t *) findDBLibSymbol(handle, "bson_new"); + Obson_append_utf8 = (Obson_append_utf8_t *) findDBLibSymbol(handle, "bson_append_utf8"); + Obson_append_date_time = (Obson_append_date_time_t *) findDBLibSymbol(handle, "bson_append_date_time"); + Obson_destroy = (Obson_destroy_t *) findDBLibSymbol(handle, "bson_destroy"); + + if ( + (Obson_new == NULL) || + (Obson_append_utf8 == NULL) || + (Obson_append_date_time == NULL) || + (Obson_destroy == NULL) + ) + { + printf("#JITServer: Failed to load all the required bson symbols\n"); + unloadDBLib(handle); + return false; + } + + return true; + } +#endif // defined(MONGO_LOGGER) + +#if defined(CASSANDRA_LOGGER) + bool loadLibcassandraAndSymbols() + { + void *handle = NULL; + + handle = loadLibcassandra(); + if (!handle) + { + printf("#JITServer: Failed to load libcassandra.\n"); + return false; + } + + Ocass_statement_new = (Ocass_statement_new_t *) findDBLibSymbol(handle, "cass_statement_new"); + Ocass_statement_free = (Ocass_statement_free_t *) findDBLibSymbol(handle, "cass_statement_free"); + Ocass_session_execute = (Ocass_session_execute_t *) findDBLibSymbol(handle, "cass_session_execute"); + Ocass_future_error_code = (Ocass_future_error_code_t *) findDBLibSymbol(handle, "cass_future_error_code"); + Ocass_future_error_message = (Ocass_future_error_message_t *) findDBLibSymbol(handle, "cass_future_error_message"); + Ocass_future_free = (Ocass_future_free_t *) findDBLibSymbol(handle, "cass_future_free"); + Ocass_cluster_new = (Ocass_cluster_new_t *) findDBLibSymbol(handle, "cass_cluster_new"); + Ocass_session_new = (Ocass_session_new_t *) findDBLibSymbol(handle, "cass_session_new"); + Ocass_cluster_set_credentials = (Ocass_cluster_set_credentials_t *) findDBLibSymbol(handle, "cass_cluster_set_credentials"); + Ocass_cluster_set_protocol_version = (Ocass_cluster_set_protocol_version_t *) findDBLibSymbol(handle, "cass_cluster_set_protocol_version"); + Ocass_session_free = (Ocass_session_free_t *) findDBLibSymbol(handle, "cass_session_free"); + Ocass_cluster_free = (Ocass_cluster_free_t *) findDBLibSymbol(handle, "cass_cluster_free"); + Ocass_cluster_set_contact_points = (Ocass_cluster_set_contact_points_t *) findDBLibSymbol(handle, "cass_cluster_set_contact_points"); + Ocass_cluster_set_port = (Ocass_cluster_set_port_t *) findDBLibSymbol(handle, "cass_cluster_set_port"); + Ocass_session_connect = (Ocass_session_connect_t *) findDBLibSymbol(handle, "cass_session_connect"); + Ocass_statement_bind_string = (Ocass_statement_bind_string_t *) findDBLibSymbol(handle, "cass_statement_bind_string"); + Ocass_error_desc = (Ocass_error_desc_t *) findDBLibSymbol(handle, "cass_error_desc"); + Ocass_time_from_epoch = (Ocass_time_from_epoch_t *) findDBLibSymbol(handle, "cass_time_from_epoch"); + Ocass_date_from_epoch = (Ocass_date_from_epoch_t *) findDBLibSymbol(handle, "cass_date_from_epoch"); + Ocass_statement_bind_int64 = (Ocass_statement_bind_int64_t *) findDBLibSymbol(handle, "cass_statement_bind_int64"); + Ocass_statement_bind_uint32 = (Ocass_statement_bind_uint32_t *)findDBLibSymbol(handle, "cass_statement_bind_uint32"); + + + if ((Ocass_statement_free == NULL ) || + (Ocass_statement_new == NULL) || + (Ocass_session_execute == NULL) || + (Ocass_future_error_code == NULL) || + (Ocass_future_error_message == NULL) || + (Ocass_future_free == NULL) || + (Ocass_cluster_new == NULL) || + (Ocass_session_new == NULL) || + (Ocass_cluster_set_credentials == NULL) || + (Ocass_cluster_set_protocol_version == NULL) || + (Ocass_session_free == NULL) || + (Ocass_cluster_free == NULL) || + (Ocass_cluster_set_contact_points == NULL) || + (Ocass_cluster_set_port == NULL) || + (Ocass_session_connect == NULL) || + (Ocass_statement_bind_string == NULL) || + (Ocass_error_desc == NULL) || + (Ocass_time_from_epoch == NULL) || + (Ocass_date_from_epoch == NULL) || + (Ocass_statement_bind_int64 == NULL) || + (Ocass_statement_bind_uint32 == NULL) + ) + { + printf("#JITServer: Failed to load all the required cassandra symbols\n"); + unloadDBLib(handle); + return false; + } + return true; + } +#endif // defined(CASSANDRA_LOGGER); + } diff --git a/runtime/compiler/control/LoadDBLibs.hpp b/runtime/compiler/control/LoadDBLibs.hpp new file mode 100644 index 00000000000..3887b0c1aef --- /dev/null +++ b/runtime/compiler/control/LoadDBLibs.hpp @@ -0,0 +1,233 @@ +// +// Created by cmbuhler on 2020-03-11. +// + +#ifndef JITSERVER_LOADDBLIBS_HPP +#define JITSERVER_LOADDBLIBS_HPP + +#include +#include + +#if defined(MONGO_LOGGER) +/* + * libbson function pointers and typedefs + */ +typedef struct Obson_t Obson_t; + +typedef struct + { + uint32_t domain; + uint32_t code; + char message[504]; + } Obson_error_t; + +typedef Obson_t *Obson_new_t(void); + +typedef bool + Obson_append_utf8_t(Obson_t *bson_client, + const char *key, + int key_length, + const char *value, + int length); + +typedef bool + Obson_append_date_time_t(Obson_t *bson, + const char *key, + int key_length, + int64_t value); + +typedef void + Obson_destroy_t(Obson_t *bson); + +/* + * libmongoc Function Pointers and typedefs + */ +typedef void Omongoc_init_t(void); + +typedef void Omongoc_cleanup_t(void); + +typedef struct Omongoc_uri_t Omongoc_uri_t; + +typedef struct Omongoc_client_t Omongoc_client_t; + +typedef struct Omongoc_database_t Omongoc_database_t; + +typedef struct Omongoc_collection_t Omongoc_collection_t; + +typedef Omongoc_uri_t * + Omongoc_uri_new_with_error_t(const char *uri_string, + Obson_error_t *error); + +typedef Omongoc_client_t * + Omongoc_client_new_from_uri_t(const Omongoc_uri_t *uri); + +typedef bool + Omongoc_client_set_appname_t(const Omongoc_client_t *client, + const char *name); + +typedef Omongoc_database_t * + Omongoc_client_get_database_t(const Omongoc_client_t *client, + const char *database_name); + +typedef Omongoc_collection_t * + Omongoc_client_get_collection_t(const Omongoc_client_t *client, + const char *database_name, + const char *collection_name); + +typedef bool + Omongoc_collection_insert_one_t(Omongoc_collection_t *collection, + const Obson_t *document, + const Obson_t *opts, + Obson_t *reply, + Obson_error_t *error); + +typedef void Omongoc_collection_destroy_t(Omongoc_collection_t *collection); + +typedef void Omongoc_database_destroy_t(Omongoc_database_t *database); + +typedef void Omongoc_uri_destroy_t(Omongoc_uri_t *uri); + +typedef void Omongoc_client_destroy_t(Omongoc_client_t *client); +#endif // defined(MONGO_LOGGER) + +#if defined(CASSANDRA_LOGGER) +/* + * libcassandra function pointers and typedefs + */ +#define OCASS_OK 0; +typedef int64_t Ocass_int64_t; + +typedef uint32_t Ocass_uint32_t; + +typedef struct OCassCluster OCassCluster; + +typedef struct OCassSession OCassSession; + +typedef struct OCassStatement OCassStatement; + +typedef struct OCassFuture OCassFuture; + +typedef int OCassError; + +typedef OCassStatement * + Ocass_statement_new_t(const char *query, size_t parameter_count); + +typedef void Ocass_statement_free_t(OCassStatement *statement); + +typedef OCassFuture * Ocass_session_execute_t(OCassSession *session, OCassStatement * statement); + +typedef int Ocass_future_error_code_t(OCassFuture *future); + +typedef void Ocass_future_error_message_t(OCassFuture * future, const char ** message, size_t * message_length); + +typedef void Ocass_future_free_t(OCassFuture *future); + +typedef OCassCluster * Ocass_cluster_new_t(); + +typedef OCassSession * Ocass_session_new_t(); + +typedef void Ocass_cluster_set_credentials_t(OCassCluster * cluster, const char * username, const char * password); + +typedef int Ocass_cluster_set_protocol_version_t(OCassCluster *cluster, int version); + +typedef void Ocass_session_free_t(OCassSession *session); + +typedef void Ocass_cluster_free_t(OCassCluster *cluster); + +typedef int Ocass_cluster_set_contact_points_t(OCassCluster *cluster, const char * contact_points); + +typedef int Ocass_cluster_set_port_t(OCassCluster *cluster, int port); + +typedef OCassFuture * Ocass_session_connect_t(OCassSession *session, OCassCluster *cluster); + +typedef int Ocass_statement_bind_string_t(OCassStatement *statement, size_t index, const char * value); + +typedef const char * Ocass_error_desc_t(int error); + +typedef Ocass_int64_t Ocass_time_from_epoch_t(Ocass_int64_t epoch_secs); + +typedef Ocass_uint32_t Ocass_date_from_epoch_t(Ocass_int64_t epoch_secs); + +typedef int Ocass_statement_bind_uint32_t(OCassStatement *statement, size_t index, Ocass_uint32_t value); + +typedef int Ocass_statement_bind_int64_t(OCassStatement *statement, size_t index, Ocass_int64_t value); +#endif // defined(CASSANDRA_LOGGER) + +/* + * Function pointer definitions. + */ +#if defined(MONGO_LOGGER) +// mongoc and bson: +extern "C" Obson_new_t *Obson_new; +extern "C" Obson_append_utf8_t *Obson_append_utf8; +extern "C" Obson_append_date_time_t *Obson_append_date_time; +extern "C" Obson_destroy_t *Obson_destroy; + +extern "C" Omongoc_init_t *Omongoc_init; +extern "C" Omongoc_cleanup_t *Omongoc_cleanup; +extern "C" Omongoc_uri_new_with_error_t *Omongoc_uri_new_with_error; +extern "C" Omongoc_client_new_from_uri_t *Omongoc_client_new_from_uri; +extern "C" Omongoc_client_set_appname_t *Omongoc_client_set_appname; +extern "C" Omongoc_client_get_database_t *Omongoc_client_get_database; +extern "C" Omongoc_client_get_collection_t *Omongoc_client_get_collection; +extern "C" Omongoc_collection_insert_one_t *Omongoc_collection_insert_one; +extern "C" Omongoc_collection_destroy_t *Omongoc_collection_destroy; +extern "C" Omongoc_database_destroy_t *Omongoc_database_destroy; +extern "C" Omongoc_uri_destroy_t *Omongoc_uri_destroy; +extern "C" Omongoc_client_destroy_t *Omongoc_client_destroy; +#endif // defined(MONGO_LOGGER) +#if defined(CASSANDRA_LOGGER) +// Cassandra: +extern "C" Ocass_statement_new_t *Ocass_statement_new; +extern "C" Ocass_statement_free_t *Ocass_statement_free; +extern "C" Ocass_session_execute_t *Ocass_session_execute; +extern "C" Ocass_future_error_code_t *Ocass_future_error_code; +extern "C" Ocass_future_error_message_t *Ocass_future_error_message; +extern "C" Ocass_future_free_t *Ocass_future_free; +extern "C" Ocass_cluster_new_t *Ocass_cluster_new; +extern "C" Ocass_session_new_t *Ocass_session_new; +extern "C" Ocass_cluster_set_credentials_t *Ocass_cluster_set_credentials; +extern "C" Ocass_cluster_set_protocol_version_t *Ocass_cluster_set_protocol_version; +extern "C" Ocass_session_free_t *Ocass_session_free; +extern "C" Ocass_cluster_free_t *Ocass_cluster_free; +extern "C" Ocass_cluster_set_contact_points_t *Ocass_cluster_set_contact_points; +extern "C" Ocass_cluster_set_port_t *Ocass_cluster_set_port; +extern "C" Ocass_session_connect_t *Ocass_session_connect; +extern "C" Ocass_statement_bind_string_t *Ocass_statement_bind_string; +extern "C" Ocass_error_desc_t *Ocass_error_desc; +extern "C" Ocass_time_from_epoch_t *Ocass_time_from_epoch; +extern "C" Ocass_date_from_epoch_t *Ocass_date_from_epoch; +extern "C" Ocass_statement_bind_int64_t *Ocass_statement_bind_int64; +extern "C" Ocass_statement_bind_uint32_t *Ocass_statement_bind_uint32; +#endif // defined(CASSANDRA_LOGGER) +namespace JITServer + { +#if defined(MONGO_LOGGER) + static bool is_mongoc_init = 0; + + inline bool isMongoCInit() { return is_mongoc_init; } + + inline void initMongoC() { Omongoc_init(); is_mongoc_init = 1; } + + inline void cleanupMongoC() { Omongoc_cleanup(); is_mongoc_init = 0; } + + void *loadLibmongoc(); + + void *loadLibbson(); +#endif // defined(MONGO_LOGGER) +#if defined(CASSANDRA_LOGGER) + void *loadLibcassandra(); +#endif //defined(CASSANDRA_LOGGER) + void unloadDBLib(void *handle); + + void *findDBLibSymbol(void *handle, const char *sym); +#if defined(MONGO_LOGGER) + bool loadLibmongocAndSymbols(); + + bool loadLibbsonAndSymbols(); +#endif // defined(MONGO_LOGGER); +#if defined(CASSANDRA_LOGGER) + bool loadLibcassandraAndSymbols(); +#endif // defined(CASSANDRA_LOGGER) + } +#endif //JITSERVER_LOADDBLIBS_HPP diff --git a/runtime/compiler/control/MongoLogger.cpp b/runtime/compiler/control/MongoLogger.cpp new file mode 100644 index 00000000000..ba8be02ab0d --- /dev/null +++ b/runtime/compiler/control/MongoLogger.cpp @@ -0,0 +1,161 @@ +#include "MongoLogger.hpp" +#include +#include +#include +#include +#include +#include "LoadDBLibs.hpp" + +MongoLogger::MongoLogger(const char* databaseIP, uint32_t databasePort, const char* databaseName) + : BasePersistentLogger(databaseIP, databasePort, databaseName) + { + init(); + } + +MongoLogger::MongoLogger(const char* databaseIP, uint32_t databasePort, const char* databaseName, + const char* databaseUsername, const char* databasePassword) + : BasePersistentLogger(databaseIP, databasePort, databaseName, databaseUsername, databasePassword) + { + init(); + } + +MongoLogger::~MongoLogger() + { + //Clean up this mongoc logger. + Omongoc_collection_destroy(_collection); + Omongoc_database_destroy(_db); + Omongoc_client_destroy(_client); + Omongoc_uri_destroy(_uri); + } + +void MongoLogger::init() + { + _uri = NULL; + _collection = NULL; + _db = NULL; + _client = NULL; + } + +char * MongoLogger::constructURI() + { + // Check if we have the database name + if (strcmp(_databaseName,"") == 0) + { + return "jitserver_logs"; + } + + // Check if we have db IP and Port + if (strcmp(_databaseIP,"") == 0) + { + // No IP try localhost + _databaseIP = "127.0.0.1"; + } + if (!_databasePort) + { + // No Port try default MongoDB port + _databasePort = 27017; + } + + // Check if we have credentials + if (strcmp(_databaseUsername,"") != 0) + { + if (strcmp(_databasePassword,"") != 0) + { + snprintf(_uri_string, 256, "mongodb://%s:%s@%s:%u/?authSource=%s", _databaseUsername, _databasePassword, + _databaseIP, _databasePort, _databaseName); + } + else + { + snprintf(_uri_string, 256, "mongodb://%s@%s:%u/?authSource=%s", _databaseUsername, _databaseIP, _databasePort, + _databaseName); + } + } + else + { + snprintf(_uri_string, 256, "mongodb://%s:%u/?authSource=%s", _databaseIP, _databasePort, _databaseName); + } + + return _uri_string; + } + +bool MongoLogger::connect() + { + Obson_error_t error; + + //Validate URI. + _uri = Omongoc_uri_new_with_error(constructURI(), &error); + if (!_uri) + { + fprintf (stderr, + "JITServer: Persistent Logger failed to parse URI: %s\n"\ + "error message: %s\n", + constructURI(), + error.message); + return false; + } + + //Create a client + _client = Omongoc_client_new_from_uri(_uri); + if(!_client) + { + Omongoc_uri_destroy(_uri); + fprintf (stderr, + "JITServer: Persistent Logger failed to create client.\n"\ + "error message: %s\n", + error.message); + return false; + } + + //Register the application name so we can track it in the profile logs + //on the server if we want. + Omongoc_client_set_appname(_client, "jitserver"); + + //Get a handle on the database and collection. + _db = Omongoc_client_get_database(_client, _databaseName); + _collection = Omongoc_client_get_collection (_client, _databaseName, "logs"); + + //Mongo is designed to be always available. Thus there is no "Connection" object + //and you will find that the "Connection" is tested on every read/write. + return true; + } + +void MongoLogger::disconnect() + { + //Does not actually do anything for Mongo as noted in connect() + return; + } + +bool MongoLogger::logMethod(const char* method, uint64_t clientID, const char* logContent) + { + struct timespec t; + clock_gettime(CLOCK_REALTIME, &t); + int64_t timestamp = t.tv_sec * INT64_C(1000) + t.tv_nsec / 1000000; + char clientIDstr[20]; + snprintf(clientIDstr, 20, "%llu", clientID); + /* + * The following constructs and inserts the following JSON structure: + * { + * "method" : "method/package/methodName()", + * "client_id" : "clientid", + * "log" : "big_log", + * "timestamp" : ISODate + * } + */ + Obson_t *insert = Obson_new(); + Obson_error_t error; + Obson_append_utf8(insert, "method", -1, method, -1); + Obson_append_utf8(insert, "client_id", -1, clientIDstr, -1); + Obson_append_utf8(insert, "log", -1, logContent, -1); + Obson_append_date_time(insert, "timestamp", -1, timestamp); + + if (!Omongoc_collection_insert_one(_collection, insert, NULL, NULL, &error)) + { + fprintf(stderr, "JITServer: Mongo Logger failed to insert log.\n" + "error message: %s\n", error.message); + } + + Obson_destroy(insert); + + return true; + } + diff --git a/runtime/compiler/control/MongoLogger.hpp b/runtime/compiler/control/MongoLogger.hpp new file mode 100644 index 00000000000..ce793e3b462 --- /dev/null +++ b/runtime/compiler/control/MongoLogger.hpp @@ -0,0 +1,36 @@ +#ifndef MONGOLOGGER_HPP +#define MONGOLOGGER_HPP + +#include "BasePersistentLogger.hpp" +#include "LoadDBLibs.hpp" + +class MongoLogger : public BasePersistentLogger + { +private: + char _uri_string[256]; + Omongoc_uri_t *_uri; + Omongoc_client_t *_client; + Omongoc_database_t *_db; + Omongoc_collection_t *_collection; + + void init(); + char* constructURI(); + +public: + bool connect() override; + + void disconnect() override; + + MongoLogger(const char* databaseIP, uint32_t databasePort, const char* databaseName); + + MongoLogger(const char* databaseIP, uint32_t databasePort, const char* databaseName, + const char* databaseUsername, const char* databasePassword); + + MongoLogger(); + + ~MongoLogger(); + + bool logMethod(const char* method, uint64_t clientID, const char* logContent) override; + }; + +#endif //MONGOLOGGER_HPP diff --git a/runtime/compiler/control/rossa.cpp b/runtime/compiler/control/rossa.cpp index 4bd18649fe2..f62593628cb 100644 --- a/runtime/compiler/control/rossa.cpp +++ b/runtime/compiler/control/rossa.cpp @@ -109,6 +109,9 @@ #include "net/CommunicationStream.hpp" #include "net/ClientStream.hpp" #include "net/LoadSSLLibs.hpp" +#if defined(MONGO_LOGGER) || defined(CASSANDRA_LOGGER) +#include "control/LoadDBLibs.hpp" +#endif //defined(MONGO_LOGGER) || defined(CASSANDRA_LOGGER) #include "runtime/JITClientSession.hpp" #include "runtime/Listener.hpp" #include "runtime/JITServerStatisticsThread.hpp" @@ -1682,6 +1685,24 @@ onLoadInternal( { ((TR_JitPrivateConfig*)(jitConfig->privateConfig))->statisticsThreadObject = NULL; } + +#if defined(MONGO_LOGGER) || defined(CASSANDRA_LOGGER) +// TODO: Get Flag from command line. + if(compInfo->getPersistentInfo()->getJITServerPersistentLogging()) + { +#if defined(MONGO_LOGGER) + if(!JITServer::loadLibmongocAndSymbols() || !JITServer::loadLibbsonAndSymbols() ) + return -1; + if(!JITServer::isMongoCInit()) + { + JITServer::initMongoC(); + } +#elif defined(CASSANDRA_LOGGER) + if(!JITServer::loadLibcassandraAndSymbols()) + return -1; +#endif //defined(CASSANDRA_LOGGER) + } +#endif //defined(MONGO_LOGGER) || defined(CASSANDRA_LOGGER) } else if (compInfo->getPersistentInfo()->getRemoteCompilationMode() == JITServer::CLIENT) { diff --git a/runtime/compiler/env/J9PersistentInfo.hpp b/runtime/compiler/env/J9PersistentInfo.hpp index 053a042684c..b9a94014d66 100644 --- a/runtime/compiler/env/J9PersistentInfo.hpp +++ b/runtime/compiler/env/J9PersistentInfo.hpp @@ -133,6 +133,18 @@ class PersistentInfo : public OMR::PersistentInfoConnector _remoteCompilationMode(JITServer::NONE), _JITServerAddress("localhost"), _JITServerPort(38400), +#if defined(MONGO_LOGGER) || defined(CASSANDRA_LOGGER) + _JITServerPersistentLogging(false), + #if defined(CASSANDRA_LOGGER) + _JITServerPersistentLoggingDatabasePort(9042), + #elif defined(MONGO_LOGGER) + _JITServerPersistentLoggingDatabasePort(27017), + #endif //MONGO_LOGGER elif CASSANDRA_LOGGER + _JITServerPersistentLoggingDatabaseAddress("127.0.0.1"), + _JITServerPersistentLoggingDatabaseUsername("admin"), + _JITServerPersistentLoggingDatabaseName("jitserver_logs"), + _JITServerPersistentLoggingDatabasePassword("password"), +#endif /* defined(MONGO_LOGGER) || defined(CASSANDRA_LOGGER) */ _socketTimeoutMs(2000), _clientUID(0), #endif /* defined(J9VM_OPT_JITSERVER) */ @@ -307,6 +319,20 @@ class PersistentInfo : public OMR::PersistentInfoConnector void setJITServerPort(uint32_t port) { _JITServerPort = port; } uint64_t getClientUID() const { return _clientUID; } void setClientUID(uint64_t val) { _clientUID = val; } +#if defined(MONGO_LOGGER) || defined(CASSANDRA_LOGGER) + void setJITServerPersistentLogging(bool enable) { _JITServerPersistentLogging = enable; } + bool getJITServerPersistentLogging() const { return _JITServerPersistentLogging; } + void setJITServerPersistentLoggingDatabasePort(uint32_t port) { _JITServerPersistentLoggingDatabasePort = port; } + uint32_t getJITServerPersistentLoggingDatabasePort() const { return _JITServerPersistentLoggingDatabasePort; } + void setJITServerPersistentLoggingDatabaseAddress(char *addr) { _JITServerPersistentLoggingDatabaseAddress = addr; } + const char *getJITServerPersistentLoggingDatabaseAddress() const { return _JITServerPersistentLoggingDatabaseAddress; } + void setJITServerPersistentLoggingDatabaseUsername(char *username) { _JITServerPersistentLoggingDatabaseUsername = username; } + const char *getJITServerPersistentLoggingDatabaseUsername() const { return _JITServerPersistentLoggingDatabaseUsername; } + void setJITServerPersistentLoggingDatabasePassword(char *password) { _JITServerPersistentLoggingDatabasePassword = password; } + const char *getJITServerPersistentLoggingDatabasePassword() const { return _JITServerPersistentLoggingDatabasePassword; } + void setJITServerPersistentLoggingDatabaseName(char *name) { _JITServerPersistentLoggingDatabaseName = name; } + const char *getJITServerPersistentLoggingDatabaseName() const { return _JITServerPersistentLoggingDatabaseName; } +#endif /* defined(defined(MONGO_LOGGER) || defined(CASSANDRA_LOGGER) */ #endif /* defined(J9VM_OPT_JITSERVER) */ private: @@ -397,6 +423,14 @@ class PersistentInfo : public OMR::PersistentInfoConnector uint32_t _JITServerPort; uint32_t _socketTimeoutMs; // timeout for communication sockets used in out-of-process JIT compilation uint64_t _clientUID; +#if defined(MONGO_LOGGER) || defined(CASSANDRA_LOGGER) + bool _JITServerPersistentLogging; + const char* _JITServerPersistentLoggingDatabaseAddress; + const char* _JITServerPersistentLoggingDatabaseUsername; + const char* _JITServerPersistentLoggingDatabasePassword; + const char* _JITServerPersistentLoggingDatabaseName; + uint32_t _JITServerPersistentLoggingDatabasePort; +#endif #endif /* defined(J9VM_OPT_JITSERVER) */ }; diff --git a/runtime/compiler/env/j9methodServer.cpp b/runtime/compiler/env/j9methodServer.cpp index 74ed985b811..3166b59e7d7 100644 --- a/runtime/compiler/env/j9methodServer.cpp +++ b/runtime/compiler/env/j9methodServer.cpp @@ -172,7 +172,6 @@ TR_ResolvedJ9JITServerMethod::definingClassFromCPFieldRef(TR::Compilation *comp, auto &cache = getJ9ClassInfo(compInfoPT, _ramClass)._fieldOrStaticDefiningClassCache; cache.insert({cpIndex, resolvedClass}); } - return resolvedClass; } @@ -2292,22 +2291,19 @@ TR_ResolvedRelocatableJ9JITServerMethod::startAddressForInterpreterOfJittedMetho { return ((J9Method *)getNonPersistentIdentifier())->extra; } - TR_OpaqueClassBlock * TR_ResolvedRelocatableJ9JITServerMethod::definingClassFromCPFieldRef(TR::Compilation *comp, int32_t cpIndex, bool isStatic) { TR_OpaqueClassBlock *resolvedClass = TR_ResolvedJ9JITServerMethod::definingClassFromCPFieldRef(comp, cpIndex, isStatic); - if (resolvedClass) - { - bool valid = false; - if (comp->getOption(TR_UseSymbolValidationManager)) - valid = comp->getSymbolValidationManager()->addDefiningClassFromCPRecord(resolvedClass, cp(), cpIndex, isStatic); - else - valid = storeValidationRecordIfNecessary(comp, cp(), cpIndex, isStatic ? TR_ValidateStaticField : TR_ValidateInstanceField, ramMethod()); - if (!valid) - resolvedClass = NULL; - } + bool valid = false; + if (comp->getOption(TR_UseSymbolValidationManager)) + valid = comp->getSymbolValidationManager()->addDefiningClassFromCPRecord(resolvedClass, cp(), cpIndex, isStatic); + else + valid = storeValidationRecordIfNecessary(comp, cp(), cpIndex, isStatic ? TR_ValidateStaticField : TR_ValidateInstanceField, ramMethod()); + + if (!valid) + resolvedClass = NULL; return resolvedClass; } diff --git a/runtime/compiler/env/j9methodServer.hpp b/runtime/compiler/env/j9methodServer.hpp index 73626ca3b60..b1df7e83850 100644 --- a/runtime/compiler/env/j9methodServer.hpp +++ b/runtime/compiler/env/j9methodServer.hpp @@ -319,4 +319,4 @@ class TR_J9ServerMethod : public TR_J9Method public: TR_J9ServerMethod(TR_FrontEnd *trvm, TR_Memory *, J9Class * aClazz, uintptr_t cpIndex); }; -#endif // J9METHODSERVER_H +#endif // J9METHODSERVER_H \ No newline at end of file