Skip to content

Commit

Permalink
Launch reprocessing from the GUI
Browse files Browse the repository at this point in the history
  • Loading branch information
takluyver committed Jul 10, 2024
1 parent 83a46ec commit d2fb95e
Show file tree
Hide file tree
Showing 3 changed files with 252 additions and 2 deletions.
41 changes: 39 additions & 2 deletions damnit/gui/main_window.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,7 @@
from ..api import DataType, RunVariables
from ..backend import backend_is_running, initialize_and_start_backend
from ..backend.db import BlobTypes, DamnitDB, MsgKind, ReducedData, db_path
from ..backend.extract_data import get_context_file
from ..backend.extraction_control import process_log_path
from ..backend.extraction_control import process_log_path, ExtractionSubmitter
from ..backend.user_variables import UserEditableVariable
from ..definitions import UPDATE_BROKERS
from ..util import StatusbarStylesheet, fix_data_for_plotting, icon_path
Expand All @@ -35,6 +34,7 @@
from .open_dialog import OpenDBDialog
from .new_context_dialog import NewContextFileDialog
from .plot import Canvas, Plot
from .process import ProcessingDialog
from .table import DamnitTableModel, TableView, prettify_notation
from .user_variables import AddUserVariableDialog
from .web_viewer import PlotlyPlot, UrlSchemeHandler
Expand Down Expand Up @@ -335,6 +335,9 @@ def _create_menu_bar(self) -> None:
action_create_var.setEnabled(False)
self.context_dir_changed.connect(lambda _: action_create_var.setEnabled(True))

action_process = QtWidgets.QAction("Reprocess runs", self)
action_process.triggered.connect(self.process_runs)

action_export = QtWidgets.QAction(QtGui.QIcon(icon_path("export.png")), "&Export", self)
action_export.setStatusTip("Export to Excel/CSV")
action_export.setEnabled(False)
Expand All @@ -360,6 +363,7 @@ def _create_menu_bar(self) -> None:
)
fileMenu.addAction(action_autoconfigure)
fileMenu.addAction(action_create_var)
fileMenu.addAction(action_process)
fileMenu.addAction(action_export)
fileMenu.addAction(action_adeqt)
fileMenu.addAction(action_help)
Expand Down Expand Up @@ -890,6 +894,39 @@ def export_selection_to_zulip(self):
df.replace(["None", '<NA>', 'nan'], '', inplace=True)
self.zulip_messenger.send_table(df)

def process_runs(self):
sel_runs_by_prop = {}
for ix in self.table_view.selected_rows():
run_prop, run_num = self.table.row_to_proposal_run(ix.row())
sel_runs_by_prop.setdefault(run_prop, []).append(run_num)

if sel_runs_by_prop:
prop, sel_runs = max(sel_runs_by_prop.items(), key=lambda p: len(p[1]))
sel_runs.sort()
else:
prop = self.db.metameta.get("proposal", "")
sel_runs = []

var_ids_titles = zip(self.table.computed_columns(),
self.table.computed_columns(by_title=True))

dlg = ProcessingDialog(str(prop), sel_runs, var_ids_titles, parent=self)
if dlg.exec() == QtWidgets.QDialog.Accepted:
submitter = ExtractionSubmitter(self.context_dir, self.db)

try:
reqs = dlg.extraction_requests()
for req in reqs:
submitter.submit(req)
except Exception as e:
log.error("Error launching processing", exc_info=True)
self.show_status_message(f"Error launching processing: {e}",
10_000, stylesheet=StatusbarStylesheet.ERROR)
else:
self.show_status_message(
f"Launched processing for {len(reqs)} runs", 10_000
)

adeqt_window = None

def show_adeqt(self):
Expand Down
205 changes: 205 additions & 0 deletions damnit/gui/process.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,205 @@
import logging
import re

from PyQt5 import QtWidgets
from PyQt5.QtCore import Qt
from PyQt5.QtWidgets import QDialogButtonBox

from ..context import RunData
from ..backend.extraction_control import ExtractionRequest

log = logging.getLogger(__name__)

run_range_re = re.compile(r"(\d+)(-\d+)?$")

RUNS_MSG = "Enter run numbers & ranges e.g. '17, 20-32'"

deselected_vars = set()


def parse_run_ranges(ranges: str) -> list[int]:
res = []
for run_s in ranges.split(","):
if m := run_range_re.match(run_s.strip()):
start = int(m[1])
if m[2] is None:
end = start
else:
end = int(m[2][1:])
if start > end:
return []
res.extend(range(start, end + 1))
else:
return []

return res


def fmt_run_ranges(run_nums: list[int]) -> str:
if not run_nums:
return ""

range_starts, range_ends = [run_nums[0]], []
current_range_end = run_nums[0]
for r in run_nums[1:]:
if r > current_range_end + 1:
range_ends.append(current_range_end)
range_starts.append(r)
current_range_end = r
range_ends.append(current_range_end)

s_pieces = []
for start, end in zip(range_starts, range_ends):
if start == end:
s_pieces.append(str(start))
else:
s_pieces.append(f"{start}-{end}")

return ", ".join(s_pieces)


class ProcessingDialog(QtWidgets.QDialog):
selected_runs = ()
all_vars_selected = False
no_vars_selected = False

def __init__(self, proposal: str, runs: list[int], var_ids_titles, parent=None):
super().__init__(parent)

self.setWindowTitle("Process runs")

main_vbox = QtWidgets.QVBoxLayout()
self.setLayout(main_vbox)

hbox1 = QtWidgets.QHBoxLayout()
main_vbox.addLayout(hbox1)

grid1 = QtWidgets.QGridLayout()
hbox1.addLayout(grid1)
vbox2 = QtWidgets.QVBoxLayout()
hbox1.addLayout(vbox2)

self.edit_prop = QtWidgets.QLineEdit(proposal)
#self.edit_prop.setInputMask('999900;_') # 4-6 digits
grid1.addWidget(QtWidgets.QLabel("Proposal:"), 0, 0)
grid1.addWidget(self.edit_prop, 0, 1)

self.edit_runs = QtWidgets.QLineEdit(fmt_run_ranges(runs))
self.edit_runs.textChanged.connect(self.validate_runs)
self.runs_hint = QtWidgets.QLabel(RUNS_MSG)
self.runs_hint.setAlignment(Qt.AlignHCenter | Qt.AlignTop)
self.runs_hint.setWordWrap(True)
grid1.addWidget(QtWidgets.QLabel("Runs:"), 1, 0)
grid1.addWidget(self.edit_runs, 1, 1)
grid1.addWidget(self.runs_hint, 2, 0, 1, 2)

self.vars_list = QtWidgets.QListWidget()
vbox2.addWidget(self.vars_list)

self.btn_select_all = QtWidgets.QPushButton("Select all")
self.btn_select_all.clicked.connect(self.select_all)
self.btn_deselect_all = QtWidgets.QPushButton("Deselect all")
self.btn_deselect_all.clicked.connect(self.deselect_all)
hbox_select_btns = QtWidgets.QHBoxLayout()
hbox_select_btns.addWidget(self.btn_select_all)
hbox_select_btns.addWidget(self.btn_deselect_all)
vbox2.addLayout(hbox_select_btns)

self.dlg_buttons = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
self.dlg_buttons.button(QDialogButtonBox.Ok).setEnabled(False)
self.dlg_buttons.accepted.connect(self.accept)
self.dlg_buttons.rejected.connect(self.reject)
main_vbox.addWidget(self.dlg_buttons)

for var_id, title in var_ids_titles:
itm = QtWidgets.QListWidgetItem(title)
itm.setData(Qt.UserRole, var_id)
itm.setCheckState(Qt.Unchecked if var_id in deselected_vars else Qt.Checked)
self.vars_list.addItem(itm)

self.vars_list.itemChanged.connect(self.validate_vars)

self.validate_runs()
self.validate_vars()

self.edit_runs.setFocus()

def validate_runs(self):
self.selected_runs = runs = parse_run_ranges(self.edit_runs.text())
if runs:
self.runs_hint.setText(f"{len(runs)} runs selected")
else:
self.runs_hint.setText(RUNS_MSG)
self.validate()

def validate_vars(self):
checks = [itm.checkState() == Qt.Checked for itm in self._var_list_items()]
self.all_vars_selected = all_sel = all(checks)
self.no_vars_selected = none_sel = not any(checks)
self.btn_select_all.setEnabled(not all_sel)
self.btn_deselect_all.setEnabled(not none_sel)
self.validate()

def validate(self):
valid = bool(self.selected_runs) and not self.no_vars_selected
self.dlg_buttons.button(QDialogButtonBox.Ok).setEnabled(valid)

def _var_list_items(self):
for i in range(self.vars_list.count()):
yield self.vars_list.item(i)

def select_all(self):
for itm in self._var_list_items():
itm.setCheckState(Qt.Checked)

def deselect_all(self):
for itm in self._var_list_items():
itm.setCheckState(Qt.Unchecked)

def save_vars_selection(self):
# We save the deselected variables, so new variables are selected
global deselected_vars
deselected_vars = {
itm.data(Qt.UserRole) for itm in self._var_list_items()
if itm.checkState() == Qt.Unchecked
}

def accept(self):
self.save_vars_selection()
super().accept()

def reject(self):
self.save_vars_selection()
super().reject()

# Results (along with .selected_runs)
def proposal_num(self) -> str:
return self.edit_prop.text()

def selected_vars(self):
return [itm.data(Qt.UserRole) for itm in self._var_list_items()
if itm.checkState() == Qt.Checked]

def extraction_requests(self):
prop = int(self.proposal_num())
if self.all_vars_selected:
var_ids = ()
else:
var_ids = tuple(self.selected_vars())
l = [ExtractionRequest(r, prop, RunData.ALL, variables=var_ids)
for r in self.selected_runs]
for req in l[1:]:
req.update_vars = False
return l



if __name__ == '__main__':
app = QtWidgets.QApplication([])
dlg = ProcessingDialog("1234", [3, 4, 5, 6, 10],
[("test_var", "Test variable"), ("n_trains", "Trains")]
)
if dlg.exec() == QtWidgets.QDialog.Accepted:
print("Proposal:", dlg.proposal_num())
print("Runs:", dlg.selected_runs)
print("Variables:", dlg.selected_vars())
8 changes: 8 additions & 0 deletions damnit/gui/table.py
Original file line number Diff line number Diff line change
Expand Up @@ -478,6 +478,14 @@ def column_id(self, col_ix):
def column_title_to_id(self, title):
return self.column_id(self.find_column(title, by_title=True))

def computed_columns(self, by_title=False):
for i, col_id in enumerate(self.column_ids[5:], start=5):
if col_id not in self.editable_columns:
if by_title:
yield self.column_titles[i]
else:
yield col_id

def find_row(self, proposal, run):
return self.run_index[(proposal, run)]

Expand Down

0 comments on commit d2fb95e

Please sign in to comment.