diff --git a/jira_agile_metrics/calculators/ageinghistory.py b/jira_agile_metrics/calculators/ageinghistory.py new file mode 100644 index 0000000..93b4af8 --- /dev/null +++ b/jira_agile_metrics/calculators/ageinghistory.py @@ -0,0 +1,92 @@ +import logging +import numpy as np +import pandas as pd +import matplotlib.pyplot as plt +import seaborn as sns + +from ..calculator import Calculator +from ..utils import set_chart_style + +from .cycletime import CycleTimeCalculator + +logger = logging.getLogger(__name__) + + +class AgeingHistoryChartCalculator(Calculator): + """Draw a chart showing breakdown of states where issues spent time. + + Unlike Ageing WIP this consideres done items as well, + so you get historical reference data to better understand the + status of the current pipeline. + """ + + def run(self, today=None): + + # short circuit relatively expensive calculation if it won't be used + if not self.settings['ageing_history_chart']: + return None + + cycle_data = self.get_result(CycleTimeCalculator) + cycle_names = [s['name'] for s in self.settings['cycle']] + + # All states between "Backlog" and "Done" + active_cycle_names = cycle_names[1:-1] + + # What Pandas columns we are going to export + series = { + 'status': {'data': [], 'dtype': 'str'}, + 'age': {'data': [], 'dtype': 'float'}, + } + # Add one column per each state + # for name in active_cycle_names: + # series[name] = {'data': [], 'dtype': 'timedelta64[ns]'} + + # For each issue create one row for each state and then duration spent in that state + for idx, row in cycle_data.iterrows(): + for state in active_cycle_names: + # Duration column as pd.timedelta is filled by new cycletime calculator + duration = row[f"{state} duration"].total_seconds() / (24 * 3600) + series["status"]["data"].append(state) + series["age"]["data"].append(duration) + + data = {} + for k, v in series.items(): + data[k] = pd.Series(v['data'], dtype=v['dtype']) + + return pd.DataFrame(data, + columns=['status', 'age'] + ) + + def write(self): + output_file = self.settings['ageing_history_chart'] + if not output_file: + logger.debug("No output file specified for ageing WIP chart") + return + + chart_data = self.get_result() + + if len(chart_data.index) == 0: + logger.warning("Unable to draw ageing WIP chart with zero completed items") + return + + fig, ax = plt.subplots() + + if self.settings['ageing_history_chart_title']: + ax.set_title(self.settings['ageing_history_chart_title']) + + sns.swarmplot(x='status', y='age', data=chart_data, ax=ax) + + ax.set_xlabel("Status") + ax.set_ylabel("Age (days)") + + ax.set_xticklabels(ax.xaxis.get_majorticklabels(), rotation=90) + + _, top = ax.get_ylim() + ax.set_ylim(0, top) + + set_chart_style() + + # Write file + logger.info("Writing ageing history chart to %s", output_file) + fig.savefig(output_file, bbox_inches='tight', dpi=300) + plt.close(fig) diff --git a/jira_agile_metrics/calculators/ageingwip.py b/jira_agile_metrics/calculators/ageingwip.py index 35e4e01..708ef6c 100644 --- a/jira_agile_metrics/calculators/ageingwip.py +++ b/jira_agile_metrics/calculators/ageingwip.py @@ -55,11 +55,11 @@ def extract_age(row): ageing_wip_data.dropna(how='any', inplace=True, subset=['status', 'age']) # reorder columns so we get key, summary, status, age, and then all the cycle stages + logger.debug("Ageing WIP data is for columns %s-%s", committed_column, last_active_column) ageing_wip_data = pd.concat(( ageing_wip_data[['key', 'summary', 'status', 'age']], ageing_wip_data.loc[:, committed_column:last_active_column] ), axis=1) - return ageing_wip_data def write(self): diff --git a/jira_agile_metrics/calculators/cycleflow.py b/jira_agile_metrics/calculators/cycleflow.py new file mode 100644 index 0000000..8ff96ca --- /dev/null +++ b/jira_agile_metrics/calculators/cycleflow.py @@ -0,0 +1,122 @@ +import logging +import pandas as pd +import numpy as np +import matplotlib.pyplot as plt + +from ..calculator import Calculator +from ..utils import get_extension, set_chart_style + +from .cycletime import CycleTimeCalculator + +logger = logging.getLogger(__name__) + + +class CycleFlowCalculator(Calculator): + """Create the data to build a non-cumulate flow diagram: a DataFrame, + indexed by day, with columns containing cumulative days for each + of the items in the configured cycle. + + """ + + def run(self): + + cycle_data = self.get_result(CycleTimeCalculator) + + # Exclude backlog and done + active_cycles = self.settings["cycle"][1:-1] + + cycle_names = [s['name'] for s in active_cycles] + + return calculate_cycle_flow_data(cycle_data, cycle_names) + + def write(self): + data = self.get_result() + + if self.settings['cycle_flow_chart']: + if data is not None: + self.write_chart(data, self.settings['cycle_flow_chart']) + else: + logger.info("Did not match any entries for cycle flow chart") + else: + logger.debug("No output file specified for cycle flow chart") + + def write_chart(self, data, output_file): + + if len(data.index) == 0: + logger.warning("Cannot draw cycle flow without data") + return + + fig, ax = plt.subplots() + + ax.set_title("Cycle flow") + data.plot.area(ax=ax, stacked=True, legend=False) + ax.set_xlabel("Period of issue complete") + ax.set_ylabel("Time spent (days)") + + ax.legend(loc='center left', bbox_to_anchor=(1, 0.5)) + + set_chart_style() + + # Write file + logger.info("Writing cycle flow chart to %s", output_file) + fig.savefig(output_file, bbox_inches='tight', dpi=300) + plt.close(fig) + + +def calculate_cycle_flow_data(cycle_data, cycle_names, frequency="1M", resample_on="completed_timestamp"): + """Calculate diagram data for times spent in different cycles. + + :param cycle_data: Cycle time calculator outpu + + :param cycle_names: List of cycles includedin the flow chat + + :param frequency: Weekly, monthly, etc. + + :param resample_on: Column that is used as the base for frequency - you can switch between start and completed timestamps + """ + + # Build a dataframe of just the "duration" columns + duration_cols = [f"{cycle} duration" for cycle in cycle_names] + cfd_data = cycle_data[[resample_on] + duration_cols] + + # Remove issues that lack completion date + # https://stackoverflow.com/a/55066805/315168 + cfd_data = cfd_data[cfd_data[resample_on].notnull()] + + # Zero out missing data for cycles which issue skipped + cfd_data = cfd_data.fillna(pd.Timedelta(seconds=0)) + + # We did not have any issues with completed_timestamp, + # cannot do resample + if cfd_data.empty: + return None + + sampled = cfd_data.resample(frequency, on=resample_on).agg(np.sum) + + # + # Sample output + # Development duration Fixes duration Review duration QA duration + # completed_timestamp + # 2020-02-29 0 days 00:02:14.829000 0 days 01:21:01.586000 0 days 06:21:59.009000 1 days 13:19:26.173000 + # 2020-03-31 4 days 04:53:44.114000 0 days 19:13:43.590000 1 days 00:51:11.272000 2 days 01:54:57.958000 + # 2020-04-30 6 days 11:48:55.864000 1 days 15:48:23.789000 3 days 17:51:01.561000 10 days 11:54:59.661000 + + # Convert Panda Timedeltas to days as float + # sampled = sampled[duration_cols].apply(lambda x: float(x.item().days)) + # https://stackoverflow.com/a/54535619/315168 + sampled[duration_cols] = sampled[duration_cols] / np.timedelta64(1, 'D') + + # Fill missing values with zero duration + sampled = sampled.fillna(0) + + # Make sure we always return stacked charts in the same order + # TODO: Not 100% sure if this is needed + sampled.columns = pd.CategoricalIndex(sampled.columns.values, + ordered=True, + categories=duration_cols) + + + # Sort the columns (axis=1) by the new categorical ordering + sampled = sampled.sort_index(axis=1) + + return sampled diff --git a/jira_agile_metrics/calculators/cycleflow100.py b/jira_agile_metrics/calculators/cycleflow100.py new file mode 100644 index 0000000..3d2d21c --- /dev/null +++ b/jira_agile_metrics/calculators/cycleflow100.py @@ -0,0 +1,69 @@ +import logging +import pandas as pd +import numpy as np +import matplotlib.pyplot as plt + +from ..calculator import Calculator +from ..utils import get_extension, set_chart_style + +from .cycletime import CycleTimeCalculator +from .cycleflow import calculate_cycle_flow_data + + +logger = logging.getLogger(__name__) + + +class CycleFlow100Calculator(Calculator): + """Same as cycle flow chart, but uses 100% stacked line graph instead. + + https://stackoverflow.com/questions/29940382/100-area-plot-of-a-pandas-dataframe + """ + + def run(self): + cycle_data = self.get_result(CycleTimeCalculator) + # Exclude backlog and done + active_cycles = self.settings["cycle"][1:-1] + cycle_names = [s['name'] for s in active_cycles] + data = calculate_cycle_flow_data(cycle_data, cycle_names) + if data is not None: + # Stack cols to 100% + data = data.divide(data.sum(axis=1), axis=0) + return data + + def write(self): + data = self.get_result() + + if self.settings['cycle_flow_100_chart']: + if data is not None: + self.write_chart(data, self.settings['cycle_flow_100_chart']) + else: + logger.info("Did not match any entries for Cycle flow 100% chart") + else: + logger.debug("No output file specified for cycle flow chart") + + def write_chart(self, data, output_file): + + if len(data.index) == 0: + logger.warning("Cannot draw cycle flow without data") + return + + fig, ax = plt.subplots() + + ax.set_title("Cycle flow") + data.plot.area(ax=ax, stacked=True, legend=False) + ax.set_xlabel("Period of issue complete") + ax.set_ylabel("Time spent (%)") + + ax.legend(loc='center left', bbox_to_anchor=(1, 0.5)) + + # bottom = data[data.columns[-1]].min() + # top = data[data.columns[0]].max() + # ax.set_ylim(bottom=bottom, top=top) + + set_chart_style() + + # Write file + logger.info("Writing cycle flow chart to %s", output_file) + fig.savefig(output_file, bbox_inches='tight', dpi=300) + plt.close(fig) + diff --git a/jira_agile_metrics/calculators/cycletime.py b/jira_agile_metrics/calculators/cycletime.py index 9a85fae..58acef4 100644 --- a/jira_agile_metrics/calculators/cycletime.py +++ b/jira_agile_metrics/calculators/cycletime.py @@ -3,12 +3,26 @@ import datetime import dateutil import pandas as pd +from enum import Enum from ..calculator import Calculator -from ..utils import get_extension, to_json_string, StatusTypes +from ..utils import get_extension, to_json_string, StatusTypes, Timespans logger = logging.getLogger(__name__) + +class BackwardsTransitionHandling(Enum): + """How to handle backwards movement in the state graph.""" + + #: When an issue re-enters a state, the previous + #: calculated time is cleared + reset = "reset" + + #: When an issue re-enters a state, the new + #: time spent on the + accumulate = "accumulate" + + class CycleTimeCalculator(Calculator): """Basic cycle time data, fetched from JIRA. @@ -47,7 +61,8 @@ def run(self, now=None): self.settings['done_column'], self.settings['queries'], self.settings['query_attribute'], - now=now + now=now, + backwards_transitions=BackwardsTransitionHandling(self.settings['backwards_transitions']), ) def write(self): @@ -58,12 +73,22 @@ def write(self): return cycle_data = self.get_result() + cycle_names = [s['name'] for s in self.settings['cycle']] + cycle_duration_labels = [s['name'] + " duration" for s in self.settings['cycle']] attribute_names = sorted(self.settings['attributes'].keys()) query_attribute_names = [self.settings['query_attribute']] if self.settings['query_attribute'] else [] - header = ['ID', 'Link', 'Name'] + cycle_names + ['Type', 'Status', 'Resolution'] + attribute_names + query_attribute_names + ['Blocked Days'] - columns = ['key', 'url', 'summary'] + cycle_names + ['issue_type', 'status', 'resolution'] + attribute_names + query_attribute_names + ['blocked_days'] + header = ['ID', 'Link', 'Name'] + cycle_names + cycle_duration_labels + ['Type', 'Status', 'Resolution'] + attribute_names + query_attribute_names + ['Blocked Days'] + columns = ['key', 'url', 'summary'] + cycle_names + cycle_duration_labels + ['issue_type', 'status', 'resolution'] + attribute_names + query_attribute_names + ['blocked_days'] + + def duration_as_days(val): + """Convert pd.Timedelta to float days""" + return val.total_seconds() / (24 * 3600) + + # Format durations as days as float + for duration_col in cycle_duration_labels: + cycle_data[duration_col] = cycle_data[duration_col].apply(duration_as_days) for output_file in output_files: @@ -79,6 +104,7 @@ def write(self): else: cycle_data.to_csv(output_file, columns=columns, header=header, date_format='%Y-%m-%d', index=False) + def calculate_cycle_times( query_manager, cycle, # [{name:"", statuses:[""], type:""}] @@ -87,14 +113,20 @@ def calculate_cycle_times( done_column, # "" in `cycle` queries, # [{jql:"", value:""}] query_attribute=None, # "" - now=None + now=None, + backwards_transitions: BackwardsTransitionHandling = None, ): # Allows unit testing to use a fixed date if now is None: now = datetime.datetime.utcnow() + # Default to reset cycle time on state transition loops + if backwards_transitions is None: + backwards_transitions = BackwardsTransitionHandling.reset + cycle_names = [s['name'] for s in cycle] + cycle_duration_names = [s['name'] + " duration" for s in cycle] # For Pandas columns active_columns = cycle_names[cycle_names.index(committed_column):cycle_names.index(done_column)] cycle_lookup = {} @@ -113,15 +145,21 @@ def calculate_cycle_times( 'issue_type': {'data': [], 'dtype': 'str'}, 'summary': {'data': [], 'dtype': 'str'}, 'status': {'data': [], 'dtype': 'str'}, + 'reporter': {'data': [], 'dtype': 'str'}, + 'assignee': {'data': [], 'dtype': 'str'}, 'resolution': {'data': [], 'dtype': 'str'}, 'cycle_time': {'data': [], 'dtype': 'timedelta64[ns]'}, 'completed_timestamp': {'data': [], 'dtype': 'datetime64[ns]'}, 'blocked_days': {'data': [], 'dtype': 'int'}, + 'estimation_days': {'data': [], 'dtype': 'float'}, 'impediments': {'data': [], 'dtype': 'object'}, # list of {'start', 'end', 'status', 'flag'} } for cycle_name in cycle_names: + # In output Pandas data, describe how to map + # datetime and timedelta to Pandas columns series[cycle_name] = {'data': [], 'dtype': 'datetime64[ns]'} + series[f'{cycle_name} duration'] = {'data': [], 'dtype': 'timedelta64[ns]'} for name in attributes: series[name] = {'data': [], 'dtype': 'object'} @@ -139,12 +177,18 @@ def calculate_cycle_times( 'summary': issue.fields.summary, 'status': issue.fields.status.name, 'resolution': issue.fields.resolution.name if issue.fields.resolution else None, + 'reporter': issue.fields.reporter, + 'assignee': issue.fields.assignee, + # Note that a workign day is 8 hours, not 24 hours + 'estimation_days': issue.fields.timeoriginalestimate / (8 * 3600) if issue.fields.timeoriginalestimate else 0, 'cycle_time': None, 'completed_timestamp': None, 'blocked_days': 0, 'impediments': [] } + logger.debug("Issue %s estimation is %f days, reported by %s and being worked on by %s", issue.key, item["estimation_days"], issue.fields.reporter, issue.fields.assignee) + for name in attributes: item[name] = query_manager.resolve_attribute_value(issue, name) @@ -154,13 +198,18 @@ def calculate_cycle_times( for cycle_name in cycle_names: item[cycle_name] = None - last_status = None + last_status = None # Name of the workflow state the last snapshot was in impediment_flag = None impediment_start_status = None impediment_start = None + # Initialze mapping of cycle name -> Timespans + # Each timespan tracks enters and exit dates of a cycle + timespans = dict([(name, Timespans()) for name in cycle_names]) + # Record date of status and impediments flag changes for snapshot in query_manager.iter_changes(issue, ['status', 'Flagged']): + if snapshot.change == 'status': snapshot_cycle_step = cycle_lookup.get(snapshot.to_string.lower(), None) if snapshot_cycle_step is None: @@ -168,21 +217,34 @@ def calculate_cycle_times( unmapped_statuses.add(snapshot.to_string) continue + logger.debug("Issue state transition %s: %s -> %s (%s) at %s", issue.key, last_status, snapshot_cycle_step["name"], snapshot.to_string, snapshot.date) + + # Looks like JIRA lib dates can be both offset-naive and offset-aware + # so we just normalise here to offset-naive + # Issue state transition FB-4281: None -> Backlog (Product Backlog) at 2020-10-09 07:43:43.681000-05:00 + # Issue state transition FB-4281: Backlog -> Backlog (Ready for Dev) at 2020-10-09 07:43:56.041000 + timepoint = snapshot.date.replace(tzinfo=None) + + # Exit from the previous timespan if there was one: + if last_status: + timespans[last_status].leave(timepoint) + last_status = snapshot_cycle_step_name = snapshot_cycle_step['name'] - # Keep the first time we entered a step - if item[snapshot_cycle_step_name] is None: - item[snapshot_cycle_step_name] = snapshot.date.date() + # Track enter of a new timespan for this cycle + timespans[snapshot_cycle_step_name].enter(timepoint) # Wipe any subsequent dates, in case this was a move backwards - found_cycle_name = False - for cycle_name in cycle_names: - if not found_cycle_name and cycle_name == snapshot_cycle_step_name: - found_cycle_name = True - continue - elif found_cycle_name and item[cycle_name] is not None: - logger.info("Issue %s moved backwards to %s [JIRA: %s -> %s], wiping data for subsequent step %s", issue.key, snapshot_cycle_step_name, snapshot.from_string, snapshot.to_string, cycle_name) - item[cycle_name] = None + if backwards_transitions == BackwardsTransitionHandling.reset: + found_cycle_name = False + for cycle_name in cycle_names: + if not found_cycle_name and cycle_name == snapshot_cycle_step_name: + found_cycle_name = True + continue + elif found_cycle_name and item[cycle_name] is not None: + logger.info("Issue %s moved backwards to %s [JIRA: %s -> %s], wiping data for subsequent step %s", issue.key, snapshot_cycle_step_name, snapshot.from_string, snapshot.to_string, cycle_name) + timespans[cycle_name].reset() + elif snapshot.change == 'Flagged': if snapshot.from_string == snapshot.to_string is None: # Initial state from None -> None @@ -238,27 +300,48 @@ def calculate_cycle_times( # calculate cycle time - previous_timestamp = None committed_timestamp = None done_timestamp = None - for cycle_name in reversed(cycle_names): - if item[cycle_name] is not None: - previous_timestamp = item[cycle_name] + if timespans[committed_column].filled: + committed_timestamp = timespans[committed_column].start - if previous_timestamp is not None: - item[cycle_name] = previous_timestamp - if cycle_name == done_column: - done_timestamp = previous_timestamp - if cycle_name == committed_column: - committed_timestamp = previous_timestamp + if timespans[done_column].filled: + done_timestamp = timespans[done_column].last_start if committed_timestamp is not None and done_timestamp is not None: item['cycle_time'] = done_timestamp - committed_timestamp + assert(item['cycle_time'] >= datetime.timedelta(seconds=0)) item['completed_timestamp'] = done_timestamp + # The legacy data handling assumes columns [state name: date] so we export these, + # but we also export durations in another column. + # Raw Timespans object is not exported ATM, + # but could be added in the future if there is need for it. + for workflow_state_name, timespans in timespans.items(): + + if timespans.filled: + # Did we have any cycles for this issue + start = timespans.start.date() + duration = timespans.duration + else: + start = None + duration = None + + item[workflow_state_name] = start + item[f'{workflow_state_name} duration'] = duration + + if duration: + days = duration.total_seconds() / (24 * 3600) + else: + days = 0 + + # Try to be helpful with the logging output to + # allow effective diagnose of transition issues + logger.debug("Calculated duration for state %s, with spans %s as %f days", workflow_state_name, timespans, days) for k, v in item.items(): + # logger.debug("Adding %s %s", k, v) series[k]['data'].append(v) if len(unmapped_statuses) > 0: @@ -269,9 +352,11 @@ def calculate_cycle_times( data[k] = pd.Series(v['data'], dtype=v['dtype']) return pd.DataFrame(data, - columns=['key', 'url', 'issue_type', 'summary', 'status', 'resolution'] + + columns=['key', 'url', 'issue_type', 'summary', 'status', 'resolution', 'estimation_days', 'reporter', 'assignee'] + sorted(attributes.keys()) + ([query_attribute] if query_attribute else []) + ['cycle_time', 'completed_timestamp', 'blocked_days', 'impediments'] + - cycle_names + cycle_duration_names + + cycle_names # Must be the last item due to legacy reasons + ) diff --git a/jira_agile_metrics/calculators/debt.py b/jira_agile_metrics/calculators/debt.py index d33df34..381651d 100644 --- a/jira_agile_metrics/calculators/debt.py +++ b/jira_agile_metrics/calculators/debt.py @@ -36,7 +36,7 @@ def run(self, now=None): if not query: logger.debug("Not calculating debt chart data as no query specified") return None - + # Resolve field name to field id for later lookup priority_field = self.settings['debt_priority_field'] priority_field_id = priority_field_id = self.query_manager.field_name_to_id(priority_field) if priority_field else None @@ -75,26 +75,26 @@ def write(self): if len(chart_data.index) == 0: logger.warning("Cannot draw debt chart with zero items") return - + if self.settings['debt_chart']: self.write_debt_chart(chart_data, self.settings['debt_chart']) - + if self.settings['debt_age_chart']: self.write_debt_age_chart(chart_data, self.settings['debt_age_chart']) - + def write_debt_chart(self, chart_data, output_file): window = self.settings['debt_window'] priority_values = self.settings['debt_priority_values'] breakdown = breakdown_by_month(chart_data, 'created', 'resolved', 'key', 'priority', priority_values) - + if window: breakdown = breakdown[-window:] fig, ax = plt.subplots() - + breakdown.plot.bar(ax=ax, stacked=True) - + if self.settings['debt_chart_title']: ax.set_title(self.settings['debt_chart_title']) @@ -111,11 +111,11 @@ def write_debt_chart(self, chart_data, output_file): logger.info("Writing debt chart to %s", output_file) fig.savefig(output_file, bbox_inches='tight', dpi=300) plt.close(fig) - + def write_debt_age_chart(self, chart_data, output_file): priority_values = self.settings['debt_priority_values'] bins = self.settings['debt_age_chart_bins'] - + def generate_bin_label(v): low, high = to_bin(v, bins) return "> %d days" % (low,) if high is None else "%d-%d days" % (low, high,) @@ -131,14 +131,14 @@ def day_grouper(value): values='key', aggfunc='count' ).groupby(day_grouper).sum().reindex(bin_labels).T - + if priority_values: breakdown = breakdown.reindex(priority_values) fig, ax = plt.subplots() - + breakdown.plot.barh(ax=ax, stacked=True) - + if self.settings['debt_age_chart_title']: ax.set_title(self.settings['debt_age_chart_title']) diff --git a/jira_agile_metrics/calculators/estimationbreakdown.py b/jira_agile_metrics/calculators/estimationbreakdown.py new file mode 100644 index 0000000..c19b06a --- /dev/null +++ b/jira_agile_metrics/calculators/estimationbreakdown.py @@ -0,0 +1,132 @@ +import logging +import numpy as np +import pandas as pd +import matplotlib.pyplot as plt +import seaborn as sns + +from ..calculator import Calculator +from ..utils import set_chart_style + +from .cycletime import CycleTimeCalculator + +logger = logging.getLogger(__name__) + + +class EstimationBreakdownCalculator(Calculator): + """Visually compare developer estimation to actual cycles spent on the issue. + + Creates two stacked bar chart horizontal rows per issue. + First one is the estimation and the second one is realised. + """ + + def run(self, today=None): + """Break down raw cycle data to split rows where the first row is actual and second row is estimation.""" + # short circuit relatively expensive calculation if it won't be used + if not self.settings['estimation_breakdown_chart']: + return None + + cycle_data = self.get_result(CycleTimeCalculator) + cycle_names = [s['name'] for s in self.settings['cycle']] + + # All states between "Backlog" and "Done" + active_cycle_names = cycle_names[1:-1] + + output = [] + + def get_good_duration(val): + if pd.isnull(val): + return 0 + else: + return val.total_seconds() / (24 * 3600) + + for idx, row in cycle_data.iterrows(): + + # Crop too long summaries + label = (row["key"] + " " + row["summary"])[0:45] + + estimation_row = {"Estimation": row["estimation_days"]} + estimation_row["label"] = label + " (est)" + estimation_row["reporter"] = row["reporter"] + estimation_row["assignee"] = row["assignee"] + + actual_row = dict([(state, get_good_duration(row[f"{state} duration"])) for state in active_cycle_names]) + actual_row["label"] = label + + output.append(estimation_row) + output.append(actual_row) + + return pd.DataFrame(output, columns=['label', 'reporter', 'assignee', 'Estimation'] + active_cycle_names) + + def write(self): + output_file = self.settings['estimation_breakdown_chart'] + if not output_file: + logger.debug("No output file specified for estimation breakdown chart") + return + + chart_data = self.get_result() + + if len(chart_data.index) == 0: + logger.warning("Unable to draw estimation breakdown without items") + return + + cycle_names = [s['name'] for s in self.settings['cycle']] + + # All states between "Backlog" and "Done" + active_cycle_names = cycle_names[1:-1] + + # https://matplotlib.org/3.1.1/gallery/lines_bars_and_markers/horizontal_barchart_distribution.html#sphx-glr-gallery-lines-bars-and-markers-horizontal-barchart-distribution-py + # https://stackoverflow.com/a/61741058/315168 + chart_data.pivot( + index='label', + columns=['Estimation'] + active_cycle_names) + + fig, ax = plt.subplots(figsize=(10, len(chart_data) / 2)) + + chart_data.plot(ax=ax, kind='barh', stacked=True) + + # https://stackoverflow.com/questions/54162981/how-to-display-data-values-in-stacked-horizontal-bar-chart-in-matplotlib + for idx, row in chart_data.iterrows(): + xpos = 0 + + # print(row["label"]) + + if "(est)" in row["label"]: + # Estimation row + xpos = (row["Estimation"] or 0) + 1 + val = row["Estimation"] + if val: + if val <= 1: + label = "< 1 day" + else: + label = "{:1.0f} days".format(val) + else: + label = "(missing)" + + label += ": {} (reported), {} (assignee)".format((row["reporter"] or "-").title(), (row["assignee"] or "-").title()) + else: + # Stacked cycle time row + total = sum([0 if pd.isnull(row[state]) else row[state] for state in active_cycle_names], 0) + parts = ["{:1.0f} ({})".format(row[state], state) for state in active_cycle_names if not pd.isnull(row[state])] + label = " + ".join(parts) + label += " = {:1.0f} days".format(total) + val = total + xpos = val + 1 + + if label: + # ax.text(xpos + 1, idx - 0.05, label, color='black') + ax.text(xpos, idx - 0.1, label, color='black') + + # Remove ticket name from the estimation label + def friendly_label(label): + if "(est)" in label: + return "Estimation" + return label + + ax.set_yticklabels(map(friendly_label, chart_data["label"].tolist()), minor=False) + + set_chart_style() + + # Write file + logger.info("Writing estimation breakdown chart to %s", output_file) + fig.savefig(output_file, bbox_inches='tight', dpi=300) + plt.close(fig) diff --git a/jira_agile_metrics/cli.py b/jira_agile_metrics/cli.py index 5cf201f..ca825f6 100644 --- a/jira_agile_metrics/cli.py +++ b/jira_agile_metrics/cli.py @@ -24,7 +24,7 @@ def configure_argument_parser(): parser.add_argument('-v', dest='verbose', action='store_true', help='Verbose output') parser.add_argument('-vv', dest='very_verbose', action='store_true', help='Even more verbose output') parser.add_argument('-n', metavar='N', dest='max_results', type=int, help='Only fetch N most recently updated issues') - + parser.add_argument('--server', metavar='127.0.0.1:8080', help='Run as a web server instead of a command line tool, on the given host and/or port. The remaining options do not apply.') # Output directory @@ -37,7 +37,7 @@ def configure_argument_parser(): parser.add_argument('--http-proxy', metavar='https://proxy.local', help='URL to HTTP Proxy') parser.add_argument('--https-proxy', metavar='https://proxy.local', help='URL to HTTPS Proxy') parser.add_argument('--jira-server-version-check', type=bool, metavar='True', help='If true it will fetch JIRA server version info first to determine if some API calls are available') - + return parser def main(): @@ -52,7 +52,7 @@ def main(): def run_server(parser, args): host = None port = args.server - + if ':' in args.server: (host, port) = args.server.split(':') port = int(port) @@ -64,7 +64,7 @@ def run_command_line(parser, args): if not args.config: parser.print_usage() return - + logging.basicConfig( format='[%(asctime)s %(levelname)s] %(message)s', datefmt='%Y-%m-%d %H:%M:%S', diff --git a/jira_agile_metrics/config.py b/jira_agile_metrics/config.py index 2e7cab1..799cebc 100644 --- a/jira_agile_metrics/config.py +++ b/jira_agile_metrics/config.py @@ -17,6 +17,10 @@ from .calculators.wip import WIPChartCalculator from .calculators.netflow import NetFlowChartCalculator from .calculators.ageingwip import AgeingWIPChartCalculator +from .calculators.ageinghistory import AgeingHistoryChartCalculator +from .calculators.estimationbreakdown import EstimationBreakdownCalculator +from .calculators.cycleflow import CycleFlowCalculator +from .calculators.cycleflow100 import CycleFlow100Calculator from .calculators.forecast import BurnupForecastCalculator from .calculators.impediments import ImpedimentsCalculator from .calculators.debt import DebtCalculator @@ -35,6 +39,10 @@ WIPChartCalculator, NetFlowChartCalculator, AgeingWIPChartCalculator, + AgeingHistoryChartCalculator, + EstimationBreakdownCalculator, + CycleFlowCalculator, + CycleFlow100Calculator, BurnupForecastCalculator, ImpedimentsCalculator, DebtCalculator, @@ -133,6 +141,7 @@ def config_to_options(data, cwd=None, extended=False): 'cycle': [], 'max_results': None, 'verbose': False, + 'backwards_transitions': "reset", 'quantiles': [0.5, 0.85, 0.95], @@ -186,6 +195,13 @@ def config_to_options(data, cwd=None, extended=False): 'ageing_wip_chart': None, 'ageing_wip_chart_title': None, + 'ageing_history_chart': None, + 'ageing_history_chart_title': None, + + 'estimation_breakdown_chart': None, + 'cycle_flow_chart': None, + 'cycle_flow_100_chart': None, + 'net_flow_frequency': '1W-MON', 'net_flow_window': None, 'net_flow_chart': None, @@ -344,7 +360,10 @@ def config_to_options(data, cwd=None, extended=False): 'burnup_forecast_chart', 'wip_chart', 'ageing_wip_chart', - 'net_flow_chart', + 'ageing_history_chart', + 'estimation_breakdown_chart', + 'cycle_flow_chart', + 'cycle_flow_100_chart', 'impediments_chart', 'impediments_days_chart', 'impediments_status_chart', @@ -390,6 +409,7 @@ def config_to_options(data, cwd=None, extended=False): 'backlog_column', 'committed_column', 'done_column', + 'backwards_transitions', 'throughput_frequency', 'scatterplot_chart_title', 'histogram_chart_title', @@ -400,6 +420,7 @@ def config_to_options(data, cwd=None, extended=False): 'wip_chart_title', 'wip_frequency', 'ageing_wip_chart_title', + 'ageing_history_chart_title', 'net_flow_chart_title', 'net_flow_frequency', 'impediments_chart_title', @@ -522,4 +543,7 @@ def config_to_options(data, cwd=None, extended=False): for name, values in config['known values'].items(): options['settings']['known_values'][name] = force_list(values) + if 'backwards transitions' in config: + options['settings']['backwards_transitions'] = config['backwards transitions'] + return options diff --git a/jira_agile_metrics/config_test.py b/jira_agile_metrics/config_test.py index 8ce2af1..5da5550 100644 --- a/jira_agile_metrics/config_test.py +++ b/jira_agile_metrics/config_test.py @@ -66,6 +66,8 @@ def test_config_to_options_maximal(): Team: Team Release: Fix version/s +Backwards transitions: accumulate + Known values: Release: - "R01" @@ -90,6 +92,8 @@ def test_config_to_options_maximal(): Committed column: Committed Done column: Done + Backwards transitions: accumulate + Cycle time data: cycletime.csv Percentiles data: percentiles.csv @@ -257,6 +261,8 @@ def test_config_to_options_maximal(): 'committed_column': 'Committed', 'done_column': 'Done', + 'backwards_transitions': "accumulate", + 'quantiles': [0.1, 0.2], 'cycle_time_data': ['cycletime.csv'], diff --git a/jira_agile_metrics/conftest.py b/jira_agile_metrics/conftest.py index 169ef9a..3979f27 100644 --- a/jira_agile_metrics/conftest.py +++ b/jira_agile_metrics/conftest.py @@ -95,6 +95,7 @@ def minimal_settings(): 'backlog_column': 'Backlog', 'committed_column': 'Committed', 'done_column': 'Done', + 'backwards_transitions': 'reset', } @@ -108,6 +109,7 @@ def custom_settings(minimal_settings): 'Team': 'Team', 'Estimate': 'Size' }, + 'backwards_transitions': 'reset', 'known_values': { 'Release': ['R1', 'R3'] }, diff --git a/jira_agile_metrics/slack.py b/jira_agile_metrics/slack.py new file mode 100644 index 0000000..58333fc --- /dev/null +++ b/jira_agile_metrics/slack.py @@ -0,0 +1,59 @@ +"""Posts diagrams to Slack after the calculators have run. + +To be run as an independent command - no depency to the main application. +""" + +try: + # Optional dependency + import slack_sdk +except ImportError as e: + raise ImportError("You need to install slack-sdk package https://pypi.org/project/slack-sdk/") from e + +import sys +import os +import argparse +import logging + +from slack_sdk import WebClient +from slack_sdk.errors import SlackApiError + +logger = logging.getLogger(__name__) +logging.basicConfig( + level=logging.INFO, + format="[%(levelname)s] %(message)s", + handlers=[ + logging.StreamHandler() + ] +) + + + +def configure_argument_parser(): + """Configure an ArgumentParser that manages command line options. + """ + parser = argparse.ArgumentParser(description='Post JIRA Agile charts to slack.') + parser.add_argument('--api-key', metavar='api_key', help='Slack API key', required=True) + parser.add_argument('--channel', metavar='channel', help='Slack channel where to post', required=True) + parser.add_argument('--diagrams', metavar='diagrams', help='Slack API key', nargs='+', required=True) + parser.add_argument('--message', metavar='message', help='Message', required=True) + return parser + + +def main(): + parser = configure_argument_parser() + args = parser.parse_args() + + # https://pypi.org/project/slack-sdk/ + client = WebClient(token=args.api_key) + + for diagram in args.diagrams: + assert os.path.exists(diagram), f"Not found {diagram}" + try: + logger.info("Posting to %s", args.channel) + response = client.files_upload(channels=args.channel, initial_comment=args.message, file=diagram) + assert response["ok"] is True + except SlackApiError as e: + # You will get a SlackApiError if "ok" is False + assert e.response["ok"] is False + assert e.response["error"] # str like 'invalid_auth', 'channel_not_found' + print(f"Got an error: {e.response['error']}") diff --git a/jira_agile_metrics/utils.py b/jira_agile_metrics/utils.py index e3392a7..993733f 100644 --- a/jira_agile_metrics/utils.py +++ b/jira_agile_metrics/utils.py @@ -1,5 +1,7 @@ import datetime import os.path +from typing import List +from typing import Tuple import numpy as np import pandas as pd @@ -124,3 +126,131 @@ def to_bin(value, edges): return (previous, v) previous = v return (previous, None) + + +class Timespans: + """Track time span spent in a state. + + Timespans class tracks (enter, exit) timestamps + for an item. One item can be re-entered multiple times. + Timespan can be open-ended. + + Because some transitions may be backwards e.g. + in the case we have state transitions: + + - In development + - Code review + - QA + - QA fixes needed + - QA (again) + - QA fixes needed (again) + - QA (third time) + + Remarks: + + - Spans can have zero duration, but not negative duration + """ + + def __init__(self): + # List of timespans for an issue state + # Each item is [datetime, datetime] + # If the closing datetime is missing the item was never closed + self.spans = [] #: List[List[datetime)] + + def reset(self): + """Used by state backwards transition tracking logic.""" + self.spans = [] + + def __str__(self): + """Produce a nice readable format of spans. + + Like 2020-03-27 08:52 - 2020-03-28 20:50, 2020-04-01 13:37 - 2020-04-01 17:02, 2020-04-07 04:28 - 2020-04-10 03:26 + """ + + if not self.spans: + return "[no duration]" + + out = "" + # https://stackoverflow.com/a/49486415/315168 + elems = list(self.spans) + while elems: + s = elems.pop(0) + out += s[0].strftime("%Y-%m-%d %H:%M") + out += " - " + if len(s) == 2: + # Span has end + out += s[1].strftime("%Y-%m-%d %H:%M") + if elems: + # Not last elements + out += ", " + return out + + def enter(self, when: datetime.datetime): + assert isinstance(when, datetime.datetime) + # Check we have been closed + if self.spans: + last_span = self.spans[-1] + assert len(last_span) == 2, "The last timespan was not closed" + self.spans.append([when]) + + def leave(self, when: datetime.datetime): + assert isinstance(when, datetime.datetime) + assert self.spans, "Cannot exit without starting a span" + last_span = self.spans[-1] + assert len(last_span) == 1, "The latest span was already closed" + assert when >= last_span[0], "Span cannot end sooner it has started" + last_span.append(when) + + @property + def filled(self): + """Does this timespan have any data""" + return True if self.spans else False + + @property + def open_ended(self): + """Have we the last timespan closed or still going on?""" + if not self.spans: + return True + return len(self.spans[-1]) == 1 + + @property + def start(self): + """When timespans started - first date.""" + assert self.spans + return self.spans[0][0] + + @property + def last_start(self): + """When the last timespans started if multiple timespans.""" + assert self.spans + return self.spans[-1][0] + + @property + def end(self): + """When timespans ended - last done date.""" + assert self.spans, "No spans" + last_span = self.spans[-1] + assert len(last_span) == 2, "Open-ended span" + return last_span[1] + + @property + def duration(self) -> datetime.timedelta: + """Duration of all timespans altogether. + + TODO: Promote to a function and make now an argument to make this testable + """ + def span_duration(s): + if len(s) == 2: + return s[1] - s[0] + else: + # Still ongoing + return datetime.datetime.now() - s[0] + + return sum([span_duration(s) for s in self.spans], datetime.timedelta()) + + + + + + + diff --git a/jira_agile_metrics/utils_test.py b/jira_agile_metrics/utils_test.py index 289a1cf..1afefc8 100644 --- a/jira_agile_metrics/utils_test.py +++ b/jira_agile_metrics/utils_test.py @@ -9,7 +9,8 @@ extend_dict, breakdown_by_month, breakdown_by_month_sum_days, - to_bin + to_bin, + Timespans ) def test_extend_dict(): @@ -47,7 +48,7 @@ def test_breakdown_by_month(): breakdown = breakdown_by_month(df, 'start', 'end', 'key', 'priority', ['low', 'med', 'high']) assert list(breakdown.columns) == ['med', 'high'] - + assert list(breakdown.index) == [ pd.Timestamp(2018, 1, 1), pd.Timestamp(2018, 2, 1), @@ -75,7 +76,7 @@ def test_breakdown_by_month_open_ended(): # Note: We will get columns until the current month; assume this test is # run from June onwards ;) - + assert list(breakdown.index)[:5] == [ pd.Timestamp(2018, 1, 1), pd.Timestamp(2018, 2, 1), @@ -104,7 +105,7 @@ def test_breakdown_by_month_no_column_spec(): breakdown = breakdown_by_month(df, 'start', 'end', 'key', 'priority') assert list(breakdown.columns) == ['high', 'med'] # alphabetical - + assert list(breakdown.index) == [ pd.Timestamp(2018, 1, 1), pd.Timestamp(2018, 2, 1), @@ -129,7 +130,7 @@ def test_breakdown_by_month_none_values(): breakdown = breakdown_by_month(df, 'start', 'end', 'key', 'priority') assert list(breakdown.columns) == [None] - + assert list(breakdown.index) == [ pd.Timestamp(2018, 1, 1), pd.Timestamp(2018, 2, 1), @@ -150,7 +151,7 @@ def test_breakdown_by_month_sum_days(): breakdown = breakdown_by_month_sum_days(df, 'start', 'end', 'priority', ['low', 'med', 'high']) assert list(breakdown.columns) == ['med', 'high'] - + assert list(breakdown.index) == [ pd.Timestamp(2018, 1, 1), pd.Timestamp(2018, 2, 1), @@ -176,7 +177,7 @@ def test_breakdown_by_month_sum_days_no_column_spec(): breakdown = breakdown_by_month_sum_days(df, 'start', 'end', 'priority') assert list(breakdown.columns) == ['high', 'med'] # alphabetical - + assert list(breakdown.index) == [ pd.Timestamp(2018, 1, 1), pd.Timestamp(2018, 2, 1), @@ -205,7 +206,7 @@ def test_breakdown_by_month_sum_day_open_ended(): # Note: We will get columns until the current month; assume this test is # run from June onwards ;) - + assert list(breakdown.index)[:5] == [ pd.Timestamp(2018, 1, 1), pd.Timestamp(2018, 2, 1), @@ -234,7 +235,7 @@ def test_breakdown_by_month_sum_days_none_values(): breakdown = breakdown_by_month_sum_days(df, 'start', 'end', 'priority') assert list(breakdown.columns) == [None, 'med'] - + assert list(breakdown.index) == [ pd.Timestamp(2018, 1, 1), pd.Timestamp(2018, 2, 1), @@ -257,5 +258,28 @@ def test_to_bin(): assert to_bin(20, [10, 20, 30]) == (10, 20) assert to_bin(30, [10, 20, 30]) == (20, 30) - + assert to_bin(31, [10, 20, 30]) == (30, None) + + +def test_timespans_single(): + t = Timespans() + t.enter(datetime.datetime(2020, 1, 1)) + assert str(t) == "2020-01-01 - " + t.leave(datetime.datetime(2020, 2, 1)) + assert str(t) == "2020-01-01 - 2020-02-01" + assert t.start == datetime.datetime(2020, 1, 1) + assert t.end == datetime.datetime(2020, 2, 1) + assert t.duration == datetime.timedelta(days=31) + + +def test_timespans_multi(): + t = Timespans() + t.enter(datetime.datetime(2020, 1, 1)) + t.leave(datetime.datetime(2020, 2, 1)) + t.enter(datetime.datetime(2020, 2, 14)) + t.leave(datetime.datetime(2020, 2, 15)) + assert str(t) == "2020-01-01 - 2020-02-01, 2020-02-14 - 2020-02-15" + assert t.start == datetime.datetime(2020, 1, 1) + assert t.end == datetime.datetime(2020, 2, 15) + assert t.duration == datetime.timedelta(days=32) diff --git a/setup.py b/setup.py index 23fc004..673838d 100644 --- a/setup.py +++ b/setup.py @@ -34,6 +34,7 @@ entry_points={ 'console_scripts': [ 'jira-agile-metrics=jira_agile_metrics.cli:main', + 'jira-agile-metrics-post-slack=jira_agile_metrics.slack:main', ], }, )