diff --git a/collectoss/api/routes/auggie.py b/collectoss/api/routes/auggie.py index 4cde77084..4fd1f8f23 100644 --- a/collectoss/api/routes/auggie.py +++ b/collectoss/api/routes/auggie.py @@ -22,231 +22,6 @@ API_VERSION = 'api/unstable' -# def annotate(metadata=None, **kwargs): -# """ -# Decorates a function as being a metric -# """ -# if metadata is None: -# metadata = {} -# def decorate(func): -# if not hasattr(func, 'metadata'): -# func.metadata = {} -# metric_metadata.append(func.metadata) - -# func.metadata.update(metadata) -# if kwargs.get('endpoint_type', None): -# endpoint_type = kwargs.pop('endpoint_type') -# if endpoint_type == 'repo': -# func.metadata['repo_endpoint'] = kwargs.get('endpoint') -# else: -# func.metadata['group_endpoint'] = kwargs.get('endpoint') - -# func.metadata.update(dict(kwargs)) - -# func.metadata['metric_name'] = request.sub('_', ' ', func.__name__).title() -# func.metadata['source'] = request.sub(r'(.*\.)', '', func.__module__) -# func.metadata['ID'] = "{}-{}".format(func.metadata['source'].lower(), func.metadata['tag']) - -# return func -# return decorate - -# def add_metrics(metrics, module_name): -# # find all unbound endpoint functions objects (ones that have metadata) defined the given module_name -# # and bind them to the metrics class -# # Derek are you proud of me -# for name, obj in inspect.getmembers(sys.modules[module_name]): -# if inspect.isfunction(obj) is True: -# if hasattr(obj, 'metadata') is True: -# setattr(metrics, name, types.MethodType(obj, metrics)) - - -# #@annotate(tag='slack_login') -# def slack_login(metric, body): -# print("slack_login") - -# r = requests.get( -# url=f'https://slack.com/api/oauth.v2.access?code={body["code"]}&client_id={os.environ["AUGGIE_CLIENT_ID"]}&client_secret={os.environ["AUGGIE_CLIENT_SECRET"]}&redirect_uri=http%3A%2F%2Flocalhost%3A8080') -# data = r.json() - -# if (data["ok"]): -# print(data) -# token = data["authed_user"]["access_token"] -# team_id = data["team"]["id"] -# webclient = slack.WebClient(token=token) - -# user_response = webclient.users_identity() -# print(user_response) -# email = user_response["user"]["email"] - -# profile_name = 'collectoss' -# if os.environ.get('AUGUR_IS_PROD'): -# profile_name = 'default' -# print("Making Boto3 Session") -# client = boto3.Session(region_name='us-east-1', -# profile_name=profile_name).client('dynamodb') -# response = client.get_item( -# TableName="auggie-users", -# Key={ -# "email": {"S": '{}:{}'.format(email, team_id)} -# } -# ) - -# if ('Item' in response): -# user = response['Item'] -# print(user) - -# filteredUser = { -# "interestedRepos": user["interestedRepos"], -# "interestedGroups": user["interestedGroups"], -# "host": user["host"], -# "maxMessages": user["maxMessages"], -# "interestedInsights": user["interestedInsightTypes"] -# } - -# user_body = json.dumps({ -# 'team_id': team_id, -# 'email': email, -# 'user': filteredUser -# }) - -# print(user_body) - -# return user_body -# else: -# client.put_item( -# TableName="auggie-users", -# Item={ -# 'botToken': {'S': 'null'}, -# 'currentMessages': {'N': "0"}, -# 'maxMessages': {'N': "0"}, -# 'email': {'S': '{}:{}'.format(email, team_id)}, -# 'host': {'S': 'null'}, -# 'interestedGroups': {'L': []}, -# 'interestedRepos': {'L': []}, -# 'interestedInsightTypes': {'L': []}, -# 'teamID': {'S': team_id}, -# 'thread': {'S': 'null'}, -# 'userID': {'S': user_response['user']['id']} -# } -# ) - -# # users_response = webclient.users_list() -# # for user in users_response["members"]: -# # if "api_app_id" in user["profile"] and user["profile"]["api_app_id"] == "ASQKB8JT0": -# # im_response = webclient.conversations_open( -# # users=user["id"] -# # ) -# # print("Hopefully IM is opened") -# # channel = im_response["channel"]["id"] - -# # message_response = webclient.chat_postMessage( -# # channel=channel, -# # text="what repos?", -# # as_user="true") -# # print(message_response) - -# # ts = message_response["ts"] -# # webclient.chat_delete( -# # channel=channel, -# # ts=ts -# # ) - -# response = client.get_item( -# TableName="auggie-users", -# Key={ -# "email": {"S": '{}:{}'.format(email, team_id)} -# } -# ) - -# user = response['Item'] -# print(user) - -# filteredUser = { -# "interestedRepos": user["interestedRepos"], -# "interestedGroups": user["interestedGroups"], -# "host": user["host"], -# "maxMessages": user["maxMessages"], -# "interestedInsights": user["interestedInsightTypes"] -# } - -# user_body = json.dumps({ -# 'team_id': team_id, -# 'email': email, -# 'user': filteredUser -# }) - -# print(user_body) - -# return user_body -# else: -# return data - -# #@annotate(tag='update-auggie-user-tracking') -# def update_tracking(metric, body): -# profile_name = 'collectoss' -# if os.environ.get('AUGUR_IS_PROD'): -# profile_name = 'default' -# client = boto3.Session(region_name='us-east-1', profile_name=profile_name).client('dynamodb') -# response = client.update_item( -# TableName="auggie-users", -# Key={ -# "email": {"S": '{}:{}'.format(body["email"], body["teamID"])} -# }, -# UpdateExpression="SET interestedGroups = :valGroup, interestedRepos = :valRepo, maxMessages = :valMax, host = :valHost, interestedInsightTypes = :valInterestedInsights", -# ExpressionAttributeValues={ -# ":valGroup": { -# "L": body["groups"] -# }, -# ":valRepo": { -# "L": body["repos"] -# }, -# ":valMax": { -# "N": body["maxMessages"] -# }, -# ":valHost": { -# "S": body["host"] -# }, -# ":valInterestedInsights": { -# "L": body["insightTypes"] -# } -# }, -# ReturnValues="ALL_NEW" -# ) - -# updated_values = response['Attributes'] - -# filtered_values = { -# "interestedRepos": updated_values["interestedRepos"], -# "interestedGroups": updated_values["interestedGroups"], -# "host": updated_values["host"] -# } - -# return filtered_values - - -# #@annotate(tag='get-auggie-user') -# def get_auggie_user(metric, body): -# profile_name = 'collectoss' -# if os.environ.get('AUGUR_IS_PROD'): -# profile_name = 'default' -# client = boto3.Session(region_name='us-east-1', profile_name=profile_name).client('dynamodb') -# response = client.get_item( -# TableName="auggie-users", -# Key={ -# "email": {"S":'{}:{}'.format(body["email"],body["teamID"])} -# } -# ) -# user = response['Item'] - -# filteredUser = { -# "interestedRepos":user["interestedRepos"], -# "interestedGroups":user["interestedGroups"], -# "host":user["host"] -# } - -# return filteredUser - - @app.route('/auggie/get_user', methods=['POST']) def get_auggie_user(): # arg = [request.json] diff --git a/collectoss/application/db/engine.py b/collectoss/application/db/engine.py index 884d5a61c..78aa9cb6c 100644 --- a/collectoss/application/db/engine.py +++ b/collectoss/application/db/engine.py @@ -8,8 +8,6 @@ from sqlalchemy import create_engine, event from sqlalchemy.engine import Engine from collectoss.application.environment import SystemEnv -from collectoss.application.db.util import catch_operational_error - def parse_database_string(db_string: str) -> tuple[str,str, str, str, str]: """Parse database string into the following components: @@ -147,25 +145,4 @@ def create_database_engine(self, **kwargs): return create_database_engine(db_conn_string, **kwargs) -class EngineConnection(): - - def __init__(self, engine): - self.connection = self.get_connection(engine) - - def __enter__(self): - return self.connection - - def __exit__(self, exception_type, exception_value, exception_traceback): - - self.connection.close() - - def get_connection(self, engine): - - func = engine.connect - - return catch_operational_error(func) - - - - diff --git a/collectoss/application/db/util.py b/collectoss/application/db/util.py index 81f24ea6d..19e25e60e 100644 --- a/collectoss/application/db/util.py +++ b/collectoss/application/db/util.py @@ -42,24 +42,6 @@ def execute_session_query(query, query_type="all"): return catch_operational_error(func) - - -def convert_orm_list_to_dict_list(result): - new_list = [] - - for row in result: - row_dict = row.__dict__ - try: - del row_dict['_sa_instance_state'] - except: - pass - - new_list.append(row_dict) - - return new_list - - - def convert_type_of_value(config_dict, logger=None): data_type = config_dict["type"] diff --git a/collectoss/tasks/git/dependency_libyear_tasks/libyear_util/pypi_parser.py b/collectoss/tasks/git/dependency_libyear_tasks/libyear_util/pypi_parser.py index 11b880e04..5c549be80 100644 --- a/collectoss/tasks/git/dependency_libyear_tasks/libyear_util/pypi_parser.py +++ b/collectoss/tasks/git/dependency_libyear_tasks/libyear_util/pypi_parser.py @@ -96,12 +96,6 @@ def map_dependencies_pipfile(packages, type): deps.append(Dict) return deps - -#def parse_pipfile(file_handle): -# manifest = tomllib.load(file_handle) -# return map_dependencies_pipfile(manifest['packages'],'runtime') + #map_dependencies_pipfile(manifest['dev-packages'], 'develop') -## Erro handling Means that the parse_pipfile(...) old function is assuming the presence of a dev-packages key in the parsed Pipfile, but that key does not exist in some cases. - def parse_pipfile(file_handle): try: manifest = tomllib.load(file_handle) diff --git a/collectoss/tasks/git/util/facade_worker/facade_worker/repofetch.py b/collectoss/tasks/git/util/facade_worker/facade_worker/repofetch.py index dfb331c1d..49f7fae21 100644 --- a/collectoss/tasks/git/util/facade_worker/facade_worker/repofetch.py +++ b/collectoss/tasks/git/util/facade_worker/facade_worker/repofetch.py @@ -34,7 +34,7 @@ from sqlalchemy.orm.exc import NoResultFound from collectoss.application.db.models.data import * from collectoss.application.db.models.operations import CollectionStatus -from collectoss.application.db.util import execute_session_query, convert_orm_list_to_dict_list +from collectoss.application.db.util import execute_session_query from collectoss.application.db.lib import execute_sql, get_repo_by_repo_git from typing_extensions import deprecated diff --git a/collectoss/tasks/github/util/gh_graphql_entities.py b/collectoss/tasks/github/util/gh_graphql_entities.py index bb5f95e98..8023c7e76 100644 --- a/collectoss/tasks/github/util/gh_graphql_entities.py +++ b/collectoss/tasks/github/util/gh_graphql_entities.py @@ -531,64 +531,3 @@ def get_pull_requests_collection(self): pull_request_collection = GraphQlPageCollection(query, self.keyAuth,self.logger,bind=params,repaginateIfIncomplete=repaginateIfIncomplete) return pull_request_collection - - - -class PullRequest(): - def __init__(self, logger, key_auth, owner, repo, number): - - self.keyAuth = key_auth - self.url = "https://api.github.com/graphql" - - self.logger = logger - - self.owner = owner - self.repo = repo - self.number = number - - def get_reviews_collection(self): - - query = """ - query MyQuery($repo: String!, $owner: String!,$number: Int!, $numRecords: Int!, $cursor: String) { - repository(name: $repo, owner: $owner) { - pullRequest(number: $number) { - reviews(first: $numRecords, after: $cursor) { - edges { - node { - author { - login - url - } - body - bodyHTML - bodyText - id - createdAt - url - } - } - totalCount - pageInfo { - hasNextPage - endCursor - } - } - } - } - } - """ - - #Values specifies the dictionary values we want to return as the issue collection. - #e.g. here we get the reviews of the specified repository by pr. - values = ("repository","pullRequest","reviews") - - params = { - 'owner' : self.owner, - 'repo' : self.repo, - 'number' : self.number, - 'values' : values - } - - review_collection = GraphQlPageCollection(query, self.keyAuth, self.logger,bind=params) - - return review_collection \ No newline at end of file diff --git a/collectoss/tasks/github/util/github_data_access.py b/collectoss/tasks/github/util/github_data_access.py index 9eaf634fc..a67a2b1cd 100644 --- a/collectoss/tasks/github/util/github_data_access.py +++ b/collectoss/tasks/github/util/github_data_access.py @@ -292,13 +292,3 @@ def __add_query_params(self, url: str, additional_params: dict) -> str: updated_query = urlencode(merged_params, doseq=True) # _replace() is how you can create a new NamedTuple with a changed field return url_components._replace(query=updated_query).geturl() - - - - - - - - - - diff --git a/collectoss/tasks/util/redis_scalar.py b/collectoss/tasks/util/redis_scalar.py deleted file mode 100644 index 29a8bb85c..000000000 --- a/collectoss/tasks/util/redis_scalar.py +++ /dev/null @@ -1,38 +0,0 @@ -"""This module defines the RedisCount class. -It imports the redis_connection as redis which is a connection to the redis cache -""" -from typing import Iterable, Any, Union - -from collections.abc import MutableSequence -from collectoss.tasks.init.redis_connection import get_redis_connection -from collectoss import instance_id -from redis import exceptions -import numbers - -class RedisScalar: - - def __init__(self, scalar_name: str, default_value: int = 0, override_existing: bool = False): - - self.redis_scalar_key = f"{instance_id}_{scalar_name}" - self._scalar_name = scalar_name - - self.__value = default_value - self.redis = get_redis_connection() - - #Check redis to see if key exists in cache - if 1 != self.redis.exists(self.redis_scalar_key) or override_existing: - #Set value - self.redis.set(self.redis_scalar_key,self.__value) - else: - #else get the value - self.__value = int(float(self.redis.get(self.redis_scalar_key))) - - @property - def value(self): - return self.__value - - @value.setter - def value(self, otherVal): - if isinstance(otherVal, numbers.Number): - self.__value = otherVal - self.redis.set(self.redis_scalar_key,self.__value) diff --git a/collectoss/tasks/util/worker_util.py b/collectoss/tasks/util/worker_util.py index 7f315d5b0..1b30e66c5 100644 --- a/collectoss/tasks/util/worker_util.py +++ b/collectoss/tasks/util/worker_util.py @@ -149,68 +149,3 @@ def parse_json_from_subprocess_call(logger, subprocess_arr, cwd=None): raise MetadataException(e, f"output : {output}") return required_output - - -# def create_server(app, worker=None): -# """ Consists of AUGWOP endpoints for the broker to communicate to this worker -# Can post a new task to be added to the workers queue -# Can retrieve current status of the worker -# Can retrieve the workers config object -# """ - -# server.app.route("/AUGWOP/task", methods=['POST', 'GET']) -# def augwop_task(): -# """ AUGWOP endpoint that gets hit to add a task to the workers queue or is used to get the heartbeat/status of worker -# """ -# if request.method == 'POST': #will post a task to be added to the queue -# app.worker.logger.info("Sending to work on task: {}".format(str(request.json))) -# app.worker.task = request.json -# return Response(response=request.json, -# status=200, -# mimetype="application/json") -# if request.method == 'GET': #will retrieve the current tasks/status of the worker -# return jsonify({ -# "status": "ALIVE", -# "results_counter": app.worker.results_counter, -# "task": app.worker.task, -# }) -# return Response(response=request.json, -# status=200, -# mimetype="application/json") - -# server.app.route("/AUGWOP/heartbeat", methods=['GET']) -# def heartbeat(): -# if request.method == 'GET': -# return jsonify({ -# "status": "alive" -# }) - -# server.app.route("/AUGWOP/config") -# def augwop_config(): -# """ Retrieve worker's config -# """ -# return app.worker.config - -# class WorkerGunicornApplication(gunicorn.app.base.BaseApplication): - -# def __init__(self, app): -# self.options = { -# 'bind': '%s:%s' % (app.worker.config["host"], app.worker.config["port"]), -# 'workers': 1, -# 'errorlog': app.worker.config['server_logfile'], -# 'accesslog': app.worker.config['server_logfile'], -# 'loglevel': app.worker.config['log_level'], -# 'capture_output': app.worker.config['capture_output'] -# } - -# self.application = app -# super().__init__() - -# def load_config(self): -# config = {key: value for key, value in self.options.items() -# if key in self.cfg.settings and value is not None} -# for key, value in config.items(): -# self.cfg.set(key.lower(), value) - -# def load(self): -# return self.application