|
1 | 1 | import json |
2 | 2 | import mimetypes |
3 | | -import multiprocessing |
4 | 3 | import os |
5 | 4 | import shutil |
6 | 5 | import subprocess |
7 | 6 | from datetime import datetime |
8 | | -from functools import cache as functools_cache |
| 7 | +from functools import cache |
9 | 8 | from logging import getLogger |
10 | 9 | from pathlib import Path |
11 | 10 |
|
12 | 11 | from api.config import config |
13 | 12 | from api.models.files import Contribution, FileInfo, UploadInfo |
14 | | -from fastapi import UploadFile |
| 13 | +from fastapi import HTTPException, UploadFile |
15 | 14 |
|
16 | 15 | logger = getLogger("uvicorn.error") |
17 | 16 |
|
|
21 | 20 | ] |
22 | 21 |
|
23 | 22 |
|
24 | | -def cleanup_git_lock(repo_path: str): |
25 | | - """Kill other git processes and remove .git/index.lock if exists.""" |
26 | | - try: |
27 | | - subprocess.run(["pkill", "-f", "git"], check=False) |
28 | | - except Exception: |
29 | | - pass |
30 | | - |
31 | | - git_lock = Path(repo_path) / ".git" / "index.lock" |
32 | | - if git_lock.exists(): |
33 | | - try: |
34 | | - git_lock.unlink() |
35 | | - logger.info(f"Removed git lock file: {git_lock}") |
36 | | - except Exception as e: |
37 | | - logger.warning(f"Failed to remove git lock file: {e}") |
38 | | - |
39 | | - |
40 | | -def cmd(command: str, working_directory: str | None = None) -> bytes: |
41 | | - """Run a shell command with real-time logging.""" |
42 | | - |
43 | | - logger.info(f"Running command: {command}") |
44 | | - |
45 | | - process = subprocess.Popen( |
46 | | - command.split(), |
47 | | - stdout=subprocess.PIPE, |
48 | | - stderr=subprocess.PIPE, |
49 | | - cwd=working_directory, |
50 | | - text=True, |
51 | | - bufsize=1, |
52 | | - universal_newlines=True, |
53 | | - ) |
54 | | - |
55 | | - stdout_lines = [] |
56 | | - stderr_lines = [] |
57 | | - |
58 | | - while True: |
59 | | - stdout_line = process.stdout.readline() if process.stdout else None |
60 | | - stderr_line = process.stderr.readline() if process.stderr else None |
61 | | - |
62 | | - if stdout_line: |
63 | | - logger.info(f"STDOUT: {stdout_line.rstrip()}") |
64 | | - stdout_lines.append(stdout_line) |
65 | | - |
66 | | - if stderr_line: |
67 | | - logger.error(f"STDERR: {stderr_line.rstrip()}") |
68 | | - stderr_lines.append(stderr_line) |
69 | | - |
70 | | - if process.poll() is not None: |
71 | | - break |
72 | | - |
73 | | - remaining_stdout, remaining_stderr = process.communicate() |
74 | | - if remaining_stdout: |
75 | | - for line in remaining_stdout.splitlines(): |
76 | | - if line.strip(): |
77 | | - logger.info(f"STDOUT: {line}") |
78 | | - stdout_lines.append(line + "\n") |
79 | | - |
80 | | - if remaining_stderr: |
81 | | - for line in remaining_stderr.splitlines(): |
82 | | - if line.strip(): |
83 | | - logger.error(f"STDERR: {line}") |
84 | | - stderr_lines.append(line + "\n") |
| 23 | +@cache |
| 24 | +def get_local_file_lfs_id(file_path: Path) -> str | None: |
| 25 | + """Check if a local file is a Git LFS pointer file.""" |
| 26 | + if not file_path.exists(): |
| 27 | + return None |
85 | 28 |
|
86 | | - return_code = process.returncode |
| 29 | + try: |
| 30 | + with open(file_path, "r") as f: |
| 31 | + first_line = f.readline().strip() |
| 32 | + if first_line != "version https://git-lfs.github.com/spec/v1": |
| 33 | + return None |
87 | 34 |
|
88 | | - if return_code != 0: |
89 | | - stderr_output = "".join(stderr_lines) |
90 | | - raise Exception(f"Command failed: {command}\n{stderr_output}") |
| 35 | + second_line = f.readline().strip() |
| 36 | + if not second_line.startswith("oid sha256:"): |
| 37 | + return None |
91 | 38 |
|
92 | | - stdout_output = "".join(stdout_lines) |
93 | | - return stdout_output.strip().encode("utf-8") |
| 39 | + return second_line.split(":")[1] |
94 | 40 |
|
| 41 | + except Exception: |
| 42 | + return None |
95 | 43 |
|
96 | | -def init_lfs_data(): |
97 | | - """Initialize LFS data by cloning the repository if not already done and checking out the specified git ref.""" |
98 | 44 |
|
99 | | - if not config.LFS_GIT_REF: |
100 | | - logger.info("LFS_GIT_REF is not set. Using local data.") |
101 | | - return |
| 45 | +@cache |
| 46 | +def get_lfs_url(oid: str) -> str: |
| 47 | + """Get the download URL for a Git LFS object ID.""" |
| 48 | + if not config.LFS_USERNAME or not config.LFS_PASSWORD: |
| 49 | + raise HTTPException( |
| 50 | + status_code=401, detail="LFS credentials are not configured" |
| 51 | + ) |
102 | 52 |
|
103 | | - lfs_server_url = config.LFS_SERVER_URL.replace( |
| 53 | + url = config.LFS_SERVER_URL.replace( |
104 | 54 | "https://", f"https://{config.LFS_USERNAME}:{config.LFS_PASSWORD}@" |
105 | 55 | ) |
106 | | - credentials_line = f"{lfs_server_url}\n" |
107 | | - git_credentials_path = Path.home() / ".git-credentials" |
108 | | - with open(git_credentials_path, "a") as f: |
109 | | - f.write(credentials_line) |
110 | | - cmd("git config --global credential.helper store") |
111 | | - |
112 | | - if not os.path.exists(config.LFS_CLONED_REPO_PATH): |
113 | | - logger.info("Creating parent directories for LFS repository clone...") |
114 | | - os.makedirs(config.LFS_CLONED_REPO_PATH, exist_ok=True) |
115 | | - logger.info("Cloning LFS repository...") |
116 | | - cmd(f"git clone {config.LFS_REPO_URL} {config.LFS_CLONED_REPO_PATH}") |
117 | | - cmd( |
118 | | - f"git checkout {config.LFS_GIT_REF}", |
119 | | - working_directory=config.LFS_CLONED_REPO_PATH, |
120 | | - ) |
121 | | - cmd("git lfs pull", working_directory=config.LFS_CLONED_REPO_PATH) |
122 | | - |
123 | | - else: |
124 | | - logger.info( |
125 | | - "LFS repository already cloned. Checking out the specified git ref and pulling..." |
126 | | - ) |
127 | | - cmd("git reset --hard", working_directory=config.LFS_CLONED_REPO_PATH) |
128 | | - cmd("git clean -fdx", working_directory=config.LFS_CLONED_REPO_PATH) |
129 | | - cmd( |
130 | | - f"git checkout {config.LFS_GIT_REF}", |
131 | | - working_directory=config.LFS_CLONED_REPO_PATH, |
132 | | - ) |
133 | | - cmd("git pull", working_directory=config.LFS_CLONED_REPO_PATH) |
134 | | - cmd("git lfs pull", working_directory=config.LFS_CLONED_REPO_PATH) |
135 | 56 |
|
136 | | - logger.info("LFS data initialized.") |
| 57 | + return f"{url}/object/{oid}" |
137 | 58 |
|
138 | 59 |
|
139 | | -def _init_lfs_data_wrapper(): |
140 | | - try: |
141 | | - init_lfs_data() |
142 | | - except Exception as e: |
143 | | - logger.error(f"Failed to initialize LFS data (subprocess): {e}") |
144 | | - logger.warning("Continuing without up to date LFS data (subprocess).") |
145 | | - finally: |
146 | | - cleanup_git_lock(config.LFS_CLONED_REPO_PATH) |
147 | | - |
148 | | - |
149 | | -@functools_cache |
| 60 | +@cache |
150 | 61 | def get_local_file_content(file_path: Path) -> tuple[bytes | None, str | None]: |
151 | 62 | """Read file content and determine MIME type.""" |
152 | 63 | if not file_path.exists(): |
@@ -314,8 +225,3 @@ def update_local_upload_info_state(relative_path: str, state: str) -> None: |
314 | 225 | raise ValueError(f"Failed to update state in info file: {e}") |
315 | 226 |
|
316 | 227 | return |
317 | | - |
318 | | - |
319 | | -multiprocessing.set_start_method("fork", force=True) |
320 | | -p = multiprocessing.Process(target=_init_lfs_data_wrapper) |
321 | | -p.start() |
0 commit comments