diff --git a/.github/workflows/export-data.yml b/.github/workflows/export-data.yml new file mode 100644 index 0000000000..e3d6f89fd1 --- /dev/null +++ b/.github/workflows/export-data.yml @@ -0,0 +1,181 @@ +name: Export Ecosystem Data + +on: + # Run weekly on Sundays at midnight + schedule: + - cron: '0 0 * * 0' + # Allow manual trigger + workflow_dispatch: + # Run on pushes to master + push: + branches: [ master, main ] + paths: + - 'migrations/**' + +jobs: + export-all: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Zig + uses: goto-bus-stop/setup-zig@v2 + with: + version: 0.11.0 + + - name: Build project + run: zig build + + - name: Get current date + id: date + run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT + + - name: Export all ecosystems + run: | + ./run.sh export exports/crypto-ecosystems-full-${{ steps.date.outputs.date }}.jsonl + echo "✅ Full export completed" + + - name: Generate ecosystem list + run: | + jq -r '.eco_name' exports/crypto-ecosystems-full-${{ steps.date.outputs.date }}.jsonl | \ + sort -u > exports/ecosystem-list-${{ steps.date.outputs.date }}.txt + echo "✅ Ecosystem list generated" + + - name: Export popular ecosystems individually + run: | + mkdir -p exports/individual + + # List of major ecosystems to export separately + ecosystems=("Bitcoin" "Ethereum" "Solana" "Polkadot" "Cardano" "Avalanche" "Polygon") + + for eco in "${ecosystems[@]}"; do + if [ -d "data/ecosystems/$eco" ] || grep -q "^ecoadd $eco$" migrations/*; then + echo "Exporting $eco..." + ./run.sh export -e "$eco" "exports/individual/${eco,,}-${{ steps.date.outputs.date }}.jsonl" + fi + done + + echo "✅ Individual exports completed" + + - name: Generate statistics JSON + run: | + cat > exports/statistics-${{ steps.date.outputs.date }}.json < exports/README.md < line.trim()) + .map(line => JSON.parse(line)); + \`\`\` + + ## Format + + Each line is a JSON object with the following structure: + \`\`\`json + { + "eco_name": "Bitcoin", + "branch": ["Lightning"], + "repo_url": "https://github.com/lightningnetwork/lnd", + "tags": ["#protocol"] + } + \`\`\` + + ## License + + This data is licensed under MIT License. See LICENSE file for details. + EOF + + echo "✅ Export README generated" + + - name: Create release archive + run: | + cd exports + tar -czf crypto-ecosystems-${{ steps.date.outputs.date }}.tar.gz *.jsonl *.txt *.json *.md individual/ + cd .. + echo "✅ Archive created" + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: crypto-ecosystems-export-${{ steps.date.outputs.date }} + path: | + exports/*.jsonl + exports/*.txt + exports/*.json + exports/*.md + exports/individual/ + retention-days: 90 + + - name: Upload release archive + uses: actions/upload-artifact@v4 + with: + name: crypto-ecosystems-archive-${{ steps.date.outputs.date }} + path: exports/*.tar.gz + retention-days: 365 + + - name: Create release (on schedule or manual trigger) + if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' + uses: softprops/action-gh-release@v1 + with: + tag_name: data-export-${{ steps.date.outputs.date }} + name: Data Export ${{ steps.date.outputs.date }} + body: | + Automated weekly export of crypto ecosystem data. + + **Statistics:** + - Total Repositories: $(jq -s 'length' exports/crypto-ecosystems-full-${{ steps.date.outputs.date }}.jsonl) + - Total Ecosystems: $(jq -r '.eco_name' exports/crypto-ecosystems-full-${{ steps.date.outputs.date }}.jsonl | sort -u | wc -l) + + Download the complete archive or individual files below. + files: | + exports/crypto-ecosystems-${{ steps.date.outputs.date }}.tar.gz + exports/crypto-ecosystems-full-${{ steps.date.outputs.date }}.jsonl + exports/statistics-${{ steps.date.outputs.date }}.json + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml new file mode 100644 index 0000000000..aa0e787034 --- /dev/null +++ b/.github/workflows/validate.yml @@ -0,0 +1,222 @@ +name: Validate Crypto Ecosystems + +on: + push: + branches: [ master, main ] + pull_request: + branches: [ master, main ] + schedule: + # Run daily at 2 AM UTC to check for broken links + - cron: '0 2 * * *' + +jobs: + validate-migrations: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Setup Zig + uses: goto-bus-stop/setup-zig@v2 + with: + version: 0.11.0 + + - name: Build project + run: zig build + + - name: Run tests + run: zig build test + + - name: Validate migration files + run: | + echo "Checking migration file naming convention..." + for file in migrations/*; do + if [[ -f "$file" ]]; then + basename=$(basename "$file") + # Check if filename matches YYYY-MM-DDThhmmss_description format + if [[ ! "$basename" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{6}_.+ ]]; then + echo "❌ Invalid migration filename: $basename" + echo "Expected format: YYYY-MM-DDThhmmss_description" + exit 1 + fi + fi + done + echo "✅ All migration files follow naming convention" + + - name: Export ecosystems + run: | + ./run.sh export test_output.jsonl + if [ ! -f test_output.jsonl ]; then + echo "❌ Export failed - no output file generated" + exit 1 + fi + echo "✅ Export successful" + + - name: Validate JSON output + run: | + echo "Validating JSONL output..." + line_count=0 + while IFS= read -r line; do + line_count=$((line_count + 1)) + if ! echo "$line" | jq empty 2>/dev/null; then + echo "❌ Invalid JSON at line $line_count: $line" + exit 1 + fi + done < test_output.jsonl + echo "✅ All $line_count lines are valid JSON" + + - name: Check for duplicate repositories + run: | + echo "Checking for duplicate repository URLs..." + duplicates=$(jq -r '.repo_url' test_output.jsonl | sort | uniq -d) + if [ ! -z "$duplicates" ]; then + echo "❌ Duplicate repositories found:" + echo "$duplicates" + exit 1 + fi + echo "✅ No duplicate repositories found" + + - name: Generate statistics + run: | + echo "📊 Ecosystem Statistics:" + echo "========================" + total_repos=$(jq -s 'length' test_output.jsonl) + total_ecosystems=$(jq -r '.eco_name' test_output.jsonl | sort -u | wc -l) + echo "Total Ecosystems: $total_ecosystems" + echo "Total Repositories: $total_repos" + echo "" + echo "Top 10 Ecosystems by Repository Count:" + jq -r '.eco_name' test_output.jsonl | sort | uniq -c | sort -rn | head -10 + + - name: Upload export artifact + uses: actions/upload-artifact@v4 + with: + name: ecosystem-export + path: test_output.jsonl + retention-days: 30 + + check-links: + runs-on: ubuntu-latest + if: github.event_name == 'schedule' || github.event_name == 'pull_request' + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Zig + uses: goto-bus-stop/setup-zig@v2 + with: + version: 0.11.0 + + - name: Build and export + run: | + zig build + ./run.sh export repos.jsonl + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + + - name: Check repository accessibility + run: | + echo "Checking repository URLs..." + broken_links=() + checked=0 + + while IFS= read -r line; do + url=$(echo "$line" | jq -r '.repo_url') + checked=$((checked + 1)) + + # Rate limit: check every 2 seconds + if [ $((checked % 10)) -eq 0 ]; then + echo "Checked $checked repositories..." + sleep 2 + fi + + # Check if URL is accessible (HTTP 200 or 301/302 redirect) + status=$(curl -o /dev/null -s -w "%{http_code}" -L "$url" --max-time 10) + + if [ "$status" != "200" ] && [ "$status" != "301" ] && [ "$status" != "302" ]; then + broken_links+=("$url (HTTP $status)") + fi + done < repos.jsonl + + if [ ${#broken_links[@]} -gt 0 ]; then + echo "⚠️ Found ${#broken_links[@]} potentially broken links:" + printf '%s\n' "${broken_links[@]}" + # Don't fail the build, just warn + exit 0 + else + echo "✅ All $checked repository URLs are accessible" + fi + + lint-migrations: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Check migration syntax + run: | + echo "Linting migration files..." + errors=0 + + for file in migrations/*; do + if [[ -f "$file" ]]; then + echo "Checking $file..." + + # Check for valid keywords + while IFS= read -r line; do + # Skip comments and empty lines + if [[ "$line" =~ ^[[:space:]]*-- ]] || [[ -z "$line" ]]; then + continue + fi + + # Check if line starts with valid keyword + if [[ ! "$line" =~ ^(ecoadd|repadd|ecocon)[[:space:]] ]]; then + echo "❌ Invalid syntax in $file: $line" + errors=$((errors + 1)) + fi + + # Validate repadd format (keyword ecosystem url [tag]) + if [[ "$line" =~ ^repadd[[:space:]] ]]; then + if [[ ! "$line" =~ ^repadd[[:space:]]+[^[:space:]]+[[:space:]]+https?:// ]]; then + echo "❌ Invalid repadd format in $file: $line" + errors=$((errors + 1)) + fi + fi + done < "$file" + fi + done + + if [ $errors -gt 0 ]; then + echo "❌ Found $errors syntax errors" + exit 1 + fi + echo "✅ All migration files have valid syntax" + + security-scan: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Run Trivy security scanner + uses: aquasecurity/trivy-action@master + with: + scan-type: 'fs' + scan-ref: '.' + format: 'sarif' + output: 'trivy-results.sarif' + + - name: Upload Trivy results to GitHub Security + uses: github/codeql-action/upload-sarif@v3 + if: always() + with: + sarif_file: 'trivy-results.sarif' \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000000..70ddec72e6 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,476 @@ +\# Contributing to Crypto Ecosystems + + + +Thank you for your interest in contributing to Crypto Ecosystems! 🎉 + + + +This guide will help you add new ecosystems, repositories, and make improvements to the project. + + + +\## Table of Contents + +\- \[Getting Started](#getting-started) + +\- \[Adding New Ecosystems](#adding-new-ecosystems) + +\- \[Adding Repositories](#adding-repositories) + +\- \[Connecting Ecosystems](#connecting-ecosystems) + +\- \[Migration File Format](#migration-file-format) + +\- \[Testing Your Changes](#testing-your-changes) + +\- \[Submitting Your Contribution](#submitting-your-contribution) + + + +\## Getting Started + + + +1\. \*\*Fork the repository\*\* to your GitHub account + +2\. \*\*Clone your fork\*\* locally: + +  ```bash + +  git clone https://github.com/YOUR\_USERNAME/crypto-ecosystems.git + +  cd crypto-ecosystems + +  ``` + +3\. \*\*Create a new branch\*\* for your changes: + +  ```bash + +  git checkout -b add-new-ecosystem + +  ``` + + + +\## Adding New Ecosystems + + + +To add a new ecosystem, create a migration file in the `migrations/` directory. + + + +\### Migration File Naming Convention + +``` + +migrations/YYYY-MM-DDThhmmss\_description\_of\_your\_migration + +``` + + + +\*\*Example:\*\* + +``` + +migrations/2025-10-03T153000\_add\_solana\_ecosystem + +``` + + + +\### Migration File Content + + + +Use the Domain Specific Language (DSL) keywords: + + + +\#### 1. \*\*Add Ecosystem\*\* - `ecoadd` + +``` + +-- Add a new ecosystem + +ecoadd Solana + +``` + + + +\#### 2. \*\*Add Repository\*\* - `repadd` + +``` + +-- Add repositories to the ecosystem + +repadd Solana https://github.com/solana-labs/solana #protocol + +repadd Solana https://github.com/project-serum/serum-dex #defi + +``` + + + +\*\*Available Tags:\*\* + +\- `#protocol` - Core protocol repositories + +\- `#defi` - Decentralized Finance + +\- `#nft` - Non-Fungible Tokens + +\- `#wallet` - Wallet implementations + +\- `#developer-tool` - Development tools + +\- `#bridge` - Cross-chain bridges + +\- `#oracle` - Oracle services + +\- `#dao` - Decentralized Autonomous Organizations + + + +\#### 3. \*\*Connect Ecosystems\*\* - `ecocon` + +``` + +-- Connect sub-ecosystems to parent ecosystems + +ecocon Ethereum Uniswap + +ecocon Bitcoin Lightning + +``` + + + +\### Complete Example Migration File + + + +\*\*File:\*\* `migrations/2025-10-03T153000\_add\_solana\_ecosystem` + + + +``` + +-- Add Solana ecosystem + +ecoadd Solana + + + +-- Add core protocol repos + +repadd Solana https://github.com/solana-labs/solana #protocol + +repadd Solana https://github.com/solana-labs/solana-program-library #protocol + + + +-- Add DeFi projects + +repadd Solana https://github.com/project-serum/serum-dex #defi + +repadd Solana https://github.com/solana-labs/solana-web3.js #developer-tool + + + +-- Add sub-ecosystem + +ecoadd Serum + +repadd Serum https://github.com/project-serum/serum-ts #developer-tool + +ecocon Solana Serum + +``` + + + +\## Adding Repositories + + + +If you want to add repositories to an \*\*existing ecosystem\*\*, simply create a migration file with `repadd` commands: + + + +\*\*File:\*\* `migrations/2025-10-03T154500\_add\_ethereum\_repos` + + + +``` + +-- Add new Ethereum repositories + +repadd Ethereum https://github.com/ethereum/go-ethereum #protocol + +repadd Ethereum https://github.com/OpenZeppelin/openzeppelin-contracts #developer-tool + +repadd Ethereum https://github.com/Uniswap/v3-core #defi + +``` + + + +\## Connecting Ecosystems + + + +To establish parent-child relationships between ecosystems: + + + +``` + +-- Polygon is a sub-ecosystem of Ethereum + +ecoadd Polygon + +ecocon Ethereum Polygon + + + +-- Optimism is a Layer 2 of Ethereum + +ecoadd Optimism + +ecocon Ethereum Optimism + +``` + + + +\## Migration File Format + + + +\### Rules: + +1\. \*\*Comments\*\* start with `--` + +2\. \*\*One command per line\*\* + +3\. \*\*Datetime format:\*\* `YYYY-MM-DDThhmmss` (ISO 8601 without colons) + +4\. \*\*No special characters\*\* in filenames except underscore `\_` + +5\. \*\*Descriptive names\*\* for clarity + + + +\### Good Examples: + +``` + +migrations/2025-10-03T120000\_add\_bitcoin\_lightning + +migrations/2025-10-03T130000\_add\_defi\_protocols + +migrations/2025-10-03T140000\_connect\_layer2\_networks + +``` + + + +\### Bad Examples: + +``` + +migrations/2025-10-03\_new\_stuff + +migrations/add\_repos + +migrations/2025:10:03T12:00:00\_update + +``` + + + +\## Testing Your Changes + + + +Before submitting, test your migration: + + + +\### On Linux/Mac: + +```bash + +./run.sh export test\_output.jsonl + +``` + + + +\### On Windows: + +```bash + +\# Install Zig if not already installed + +\# Then run: + +zig build run -- export test\_output.jsonl + +``` + + + +Check if your changes appear in the output: + +```bash + +\# On Linux/Mac + +cat test\_output.jsonl | grep "your\_ecosystem\_name" + + + +\# On Windows + +findstr "your\_ecosystem\_name" test\_output.jsonl + +``` + + + +\## Submitting Your Contribution + + + +1\. \*\*Commit your changes:\*\* + +  ```bash + +  git add migrations/ + +  git commit -m "Add Solana ecosystem with core repositories" + +  ``` + + + +2\. \*\*Push to your fork:\*\* + +  ```bash + +  git push origin add-new-ecosystem + +  ``` + + + +3\. \*\*Create a Pull Request:\*\* + +  - Go to the original repository on GitHub + +  - Click "New Pull Request" + +  - Select your branch + +  - Fill in the PR template with: + +  - \*\*Description:\*\* What ecosystems/repos you added + +  - \*\*Motivation:\*\* Why they should be included + +  - \*\*Testing:\*\* Confirm you tested the migration + + + +4\. \*\*PR Title Format:\*\* + +  ``` + +  Add \[Ecosystem Name] ecosystem + +  Add repositories to \[Ecosystem Name] + +  Connect \[Child] to \[Parent] ecosystem + +  ``` + + + +\## Quality Guidelines + + + +\### Repository Requirements: + +\- ✅ Must be \*\*open source\*\* + +\- ✅ Must be related to \*\*blockchain/crypto/web3\*\* + +\- ✅ Must be \*\*actively maintained\*\* (or historically significant) + +\- ✅ Must have a \*\*valid GitHub URL\*\* + +\- ❌ No private repositories + +\- ❌ No spam or low-quality projects + + + +\### Ecosystem Requirements: + +\- ✅ Must be a \*\*recognized blockchain ecosystem\*\* + +\- ✅ Must have multiple \*\*active repositories\*\* + +\- ✅ Must be \*\*publicly documented\*\* + + + +\## Need Help? + + + +\- 📖 Check the \[README.md](README.md) for usage examples + +\- 🐛 Report bugs in \[Issues](https://github.com/PROFADAM/crypto-ecosystems/issues) + +\- 💬 Ask questions in \[Discussions](https://github.com/PROFADAM/crypto-ecosystems/discussions) + +\- 📧 Contact: \[Your Contact Info] + + + +\## Code of Conduct + + + +\- Be respectful and constructive + +\- Focus on the technology, not personal opinions + +\- Provide accurate information + +\- Help others learn and contribute + + + +\## License + + + +By contributing, you agree that your contributions will be licensed under the MIT License. + + + +--- + + + +Thank you for helping build the most comprehensive crypto ecosystem taxonomy! 🚀 + diff --git a/README.md b/README.md index c19f437409..851060113f 100644 --- a/README.md +++ b/README.md @@ -56,8 +56,7 @@ ecoadd Lightning repadd Lightning https://github.com/lightningnetwork/lnd #protocol -- Connect ecosystems using the ecocon keyword. -- The following connects Lighting as a sub ecosystem of Bitcoin. -ecocon Bitcoin Lighting -``` +ecocon Bitcoin Lightning``` ## How to Give Attribution For Usage of the Electric Capital Crypto Ecosystems diff --git a/api/README.md b/api/README.md new file mode 100644 index 0000000000..7ef1659aca --- /dev/null +++ b/api/README.md @@ -0,0 +1,622 @@ +\# Crypto Ecosystems REST API + + + +A Flask-based REST API for accessing crypto ecosystem taxonomy data. + + + +\## Installation + + + +\### Prerequisites + +\- Python 3.8 or higher + +\- Zig (for building the main project) + + + +\### Setup + + + +1\. \*\*Install Python dependencies:\*\* + +  ```bash + +  cd api + +  pip install -r requirements.txt + +  ``` + + + +2\. \*\*Build the main project (required for data export):\*\* + +  ```bash + +  cd .. + +  zig build + +  ``` + + + +\## Running the API + + + +\### Development Mode + +```bash + +cd api + +python app.py + +``` + + + +The API will start at `http://localhost:5000` + + + +\### Production Mode (with Gunicorn) + +```bash + +pip install gunicorn + +gunicorn -w 4 -b 0.0.0.0:5000 app:app + +``` + + + +\## API Endpoints + + + +\### 📚 Documentation + +``` + +GET / + +``` + +Returns API documentation and available endpoints. + + + +\### 🌐 Get All Ecosystems + +``` + +GET /api/ecosystems + +``` + +Returns a list of all ecosystems with repository counts. + + + +\*\*Response:\*\* + +```json + +{ + +  "total": 150, + +  "ecosystems": \[ + +  { + +  "name": "Bitcoin", + +  "repository\_count": 523, + +  "sub\_ecosystem\_count": 5, + +  "sub\_ecosystems": \["Lightning", "Liquid"] + +  } + +  ] + +} + +``` + + + +\### 🔍 Get Specific Ecosystem + +``` + +GET /api/ecosystems/ + +``` + +Returns detailed information about a specific ecosystem. + + + +\*\*Example:\*\* + +```bash + +curl http://localhost:5000/api/ecosystems/Bitcoin + +``` + + + +\*\*Response:\*\* + +```json + +{ + +  "name": "Bitcoin", + +  "repository\_count": 523, + +  "repositories": \["https://github.com/bitcoin/bitcoin", ...], + +  "sub\_ecosystems": \["Lightning", "Liquid"], + +  "tags": { + +  "#protocol": 45, + +  "#wallet": 32, + +  "#developer-tool": 28 + +  }, + +  "sample\_repos": \[...] + +} + +``` + + + +\### 📦 Get All Repositories + +``` + +GET /api/repositories?page=1\&per\_page=50 + +``` + +Returns paginated list of all repositories. + + + +\*\*Query Parameters:\*\* + +\- `page` (default: 1) - Page number + +\- `per\_page` (default: 50, max: 200) - Items per page + + + +\*\*Response:\*\* + +```json + +{ + +  "total": 15234, + +  "page": 1, + +  "per\_page": 50, + +  "total\_pages": 305, + +  "repositories": \[...] + +} + +``` + + + +\### 🔎 Search Repositories + +``` + +GET /api/repositories/search?q= + +``` + +Search repositories by name, ecosystem, or tags. + + + +\*\*Example:\*\* + +```bash + +curl http://localhost:5000/api/repositories/search?q=ethereum + +``` + + + +\*\*Response:\*\* + +```json + +{ + +  "query": "ethereum", + +  "total": 234, + +  "results": \[...] + +} + +``` + + + +\### 🏷️ Get All Tags + +``` + +GET /api/tags + +``` + +Returns all available tags with their usage counts. + + + +\*\*Response:\*\* + +```json + +{ + +  "total\_tags": 15, + +  "tags": \[ + +  {"tag": "#protocol", "count": 423}, + +  {"tag": "#defi", "count": 312}, + +  {"tag": "#wallet", "count": 245} + +  ] + +} + +``` + + + +\### 📊 Get Statistics + +``` + +GET /api/stats + +``` + +Returns overall statistics about the dataset. + + + +\*\*Response:\*\* + +```json + +{ + +  "timestamp": "2025-10-03T16:30:00", + +  "total\_ecosystems": 156, + +  "total\_repositories": 15234, + +  "total\_tags": 15, + +  "top\_ecosystems": \[ + +  {"name": "Ethereum", "repository\_count": 3245}, + +  {"name": "Bitcoin", "repository\_count": 1523} + +  ], + +  "tag\_distribution": \[...], + +  "data\_freshness": { + +  "cached": true, + +  "cache\_age\_seconds": 245 + +  } + +} + +``` + + + +\### ❤️ Health Check + +``` + +GET /api/health + +``` + +Returns API health status. + + + +\*\*Response:\*\* + +```json + +{ + +  "status": "healthy", + +  "timestamp": "2025-10-03T16:30:00", + +  "cache\_age\_seconds": 245 + +} + +``` + + + +\## Usage Examples + + + +\### Python + +```python + +import requests + + + +\# Get all ecosystems + +response = requests.get('http://localhost:5000/api/ecosystems') + +ecosystems = response.json() + + + +\# Search for Ethereum repositories + +response = requests.get('http://localhost:5000/api/repositories/search?q=ethereum') + +results = response.json() + + + +\# Get statistics + +response = requests.get('http://localhost:5000/api/stats') + +stats = response.json() + +print(f"Total repos: {stats\['total\_repositories']}") + +``` + + + +\### JavaScript (Node.js) + +```javascript + +const axios = require('axios'); + + + +async function getEcosystems() { + +  const response = await axios.get('http://localhost:5000/api/ecosystems'); + +  console.log(response.data); + +} + + + +async function searchRepos(query) { + +  const response = await axios.get( + +  `http://localhost:5000/api/repositories/search?q=${query}` + +  ); + +  return response.data.results; + +} + +``` + + + +\### cURL + +```bash + +\# Get all ecosystems + +curl http://localhost:5000/api/ecosystems + + + +\# Get Bitcoin ecosystem details + +curl http://localhost:5000/api/ecosystems/Bitcoin + + + +\# Search for DeFi projects + +curl "http://localhost:5000/api/repositories/search?q=defi" + + + +\# Get statistics + +curl http://localhost:5000/api/stats + +``` + + + +\## Caching + + + +The API caches exported data for 1 hour (3600 seconds) to improve performance. After the cache expires, data is automatically refreshed from the latest migrations. + + + +\## CORS + + + +CORS is enabled for all origins, making the API accessible from web applications. + + + +\## Error Handling + + + +The API returns standard HTTP status codes: + +\- `200` - Success + +\- `400` - Bad Request (missing or invalid parameters) + +\- `404` - Not Found (ecosystem or endpoint doesn't exist) + +\- `500` - Internal Server Error + + + +Error responses follow this format: + +```json + +{ + +  "error": "Error description" + +} + +``` + + + +\## Docker Deployment + + + +Create a `Dockerfile`: + +```dockerfile + +FROM python:3.11-slim + + + +WORKDIR /app + + + +\# Install Zig + +RUN apt-get update \&\& apt-get install -y wget xz-utils + +RUN wget https://ziglang.org/download/0.11.0/zig-linux-x86\_64-0.11.0.tar.xz + +RUN tar -xf zig-linux-x86\_64-0.11.0.tar.xz + +ENV PATH="/app/zig-linux-x86\_64-0.11.0:${PATH}" + + + +\# Copy project files + +COPY . . + + + +\# Build project + +RUN zig build + + + +\# Install Python dependencies + +RUN pip install -r api/requirements.txt + + + +EXPOSE 5000 + + + +CMD \["python", "api/app.py"] + +``` + + + +Build and run: + +```bash + +docker build -t crypto-ecosystems-api . + +docker run -p 5000:5000 crypto-ecosystems-api + +``` + + + +\## Contributing + + + +See \[CONTRIBUTING.md](../CONTRIBUTING.md) for guidelines on contributing to this project. + + + +\## License + + + +MIT License - See \[LICENSE](../LICENSE) for details. + diff --git a/api/app.py b/api/app.py new file mode 100644 index 0000000000..f829344a10 --- /dev/null +++ b/api/app.py @@ -0,0 +1,272 @@ +from flask import Flask, jsonify, request +from flask_cors import CORS +import json +import os +import subprocess +from collections import defaultdict, Counter +from datetime import datetime + +app = Flask(__name__) +CORS(app) # Enable CORS for all routes + +# Cache for exported data +_cache = { + 'data': None, + 'timestamp': None, + 'ttl': 3600 # Cache for 1 hour +} + +def load_ecosystem_data(): + """Load ecosystem data from export or cache""" + current_time = datetime.now() + + # Return cached data if still valid + if _cache['data'] and _cache['timestamp']: + time_diff = (current_time - _cache['timestamp']).seconds + if time_diff < _cache['ttl']: + return _cache['data'] + + # Generate fresh export + export_file = 'api_export.jsonl' + + try: + # Run export command + if os.name == 'nt': # Windows + subprocess.run(['cmd', '/c', 'run.sh', 'export', export_file], check=True) + else: # Linux/Mac + subprocess.run(['./run.sh', 'export', export_file], check=True) + + # Load data + data = [] + with open(export_file, 'r', encoding='utf-8') as f: + for line in f: + if line.strip(): + data.append(json.loads(line)) + + # Update cache + _cache['data'] = data + _cache['timestamp'] = current_time + + # Clean up export file + os.remove(export_file) + + return data + except Exception as e: + print(f"Error loading data: {e}") + return _cache['data'] if _cache['data'] else [] + +@app.route('/') +def index(): + """API documentation""" + return jsonify({ + 'name': 'Crypto Ecosystems API', + 'version': '1.0.0', + 'description': 'REST API for crypto ecosystem taxonomy data', + 'endpoints': { + '/': 'API documentation (this page)', + '/api/ecosystems': 'List all ecosystems', + '/api/ecosystems/': 'Get specific ecosystem details', + '/api/repositories': 'List all repositories', + '/api/repositories/search?q=': 'Search repositories', + '/api/stats': 'Get overall statistics', + '/api/tags': 'List all available tags', + '/api/health': 'API health check' + }, + 'documentation': 'https://github.com/PROFADAM/crypto-ecosystems' + }) + +@app.route('/api/health') +def health(): + """Health check endpoint""" + return jsonify({ + 'status': 'healthy', + 'timestamp': datetime.now().isoformat(), + 'cache_age_seconds': (datetime.now() - _cache['timestamp']).seconds if _cache['timestamp'] else None + }) + +@app.route('/api/ecosystems') +def get_ecosystems(): + """Get list of all ecosystems with their repository counts""" + data = load_ecosystem_data() + + # Group by ecosystem + ecosystems = defaultdict(lambda: {'repos': [], 'sub_ecosystems': set()}) + + for item in data: + eco_name = item['eco_name'] + ecosystems[eco_name]['repos'].append(item['repo_url']) + + # Track sub-ecosystems + if item.get('branch'): + for branch in item['branch']: + ecosystems[eco_name]['sub_ecosystems'].add(branch) + + # Format response + result = [] + for name, info in sorted(ecosystems.items()): + result.append({ + 'name': name, + 'repository_count': len(info['repos']), + 'sub_ecosystem_count': len(info['sub_ecosystems']), + 'sub_ecosystems': sorted(list(info['sub_ecosystems'])) + }) + + return jsonify({ + 'total': len(result), + 'ecosystems': result + }) + +@app.route('/api/ecosystems/') +def get_ecosystem(ecosystem_name): + """Get detailed information about a specific ecosystem""" + data = load_ecosystem_data() + + # Filter by ecosystem name (case-insensitive) + ecosystem_data = [ + item for item in data + if item['eco_name'].lower() == ecosystem_name.lower() + ] + + if not ecosystem_data: + return jsonify({'error': 'Ecosystem not found'}), 404 + + # Collect statistics + repos = [item['repo_url'] for item in ecosystem_data] + tags = [] + for item in ecosystem_data: + tags.extend(item.get('tags', [])) + + sub_ecosystems = set() + for item in ecosystem_data: + if item.get('branch'): + sub_ecosystems.update(item['branch']) + + return jsonify({ + 'name': ecosystem_data[0]['eco_name'], + 'repository_count': len(repos), + 'repositories': repos, + 'sub_ecosystems': sorted(list(sub_ecosystems)), + 'tags': dict(Counter(tags)), + 'sample_repos': repos[:10] # First 10 repos as sample + }) + +@app.route('/api/repositories') +def get_repositories(): + """Get all repositories with pagination""" + data = load_ecosystem_data() + + # Pagination + page = request.args.get('page', 1, type=int) + per_page = request.args.get('per_page', 50, type=int) + per_page = min(per_page, 200) # Max 200 per page + + start = (page - 1) * per_page + end = start + per_page + + total = len(data) + repos = data[start:end] + + return jsonify({ + 'total': total, + 'page': page, + 'per_page': per_page, + 'total_pages': (total + per_page - 1) // per_page, + 'repositories': repos + }) + +@app.route('/api/repositories/search') +def search_repositories(): + """Search repositories by query""" + query = request.args.get('q', '').lower() + + if not query: + return jsonify({'error': 'Query parameter "q" is required'}), 400 + + data = load_ecosystem_data() + + # Search in repo URL and ecosystem name + results = [ + item for item in data + if query in item['repo_url'].lower() or + query in item['eco_name'].lower() or + any(query in tag.lower() for tag in item.get('tags', [])) + ] + + return jsonify({ + 'query': query, + 'total': len(results), + 'results': results + }) + +@app.route('/api/tags') +def get_tags(): + """Get all available tags with their counts""" + data = load_ecosystem_data() + + tags = [] + for item in data: + tags.extend(item.get('tags', [])) + + tag_counts = Counter(tags) + + return jsonify({ + 'total_tags': len(tag_counts), + 'tags': [ + {'tag': tag, 'count': count} + for tag, count in tag_counts.most_common() + ] + }) + +@app.route('/api/stats') +def get_stats(): + """Get overall statistics""" + data = load_ecosystem_data() + + # Calculate statistics + ecosystems = set(item['eco_name'] for item in data) + repos = set(item['repo_url'] for item in data) + + tags = [] + for item in data: + tags.extend(item.get('tags', [])) + + # Top ecosystems by repo count + eco_counts = Counter(item['eco_name'] for item in data) + top_ecosystems = [ + {'name': name, 'repository_count': count} + for name, count in eco_counts.most_common(10) + ] + + # Tag distribution + tag_counts = Counter(tags) + tag_distribution = [ + {'tag': tag, 'count': count} + for tag, count in tag_counts.most_common(10) + ] + + return jsonify({ + 'timestamp': datetime.now().isoformat(), + 'total_ecosystems': len(ecosystems), + 'total_repositories': len(repos), + 'total_tags': len(set(tags)), + 'top_ecosystems': top_ecosystems, + 'tag_distribution': tag_distribution, + 'data_freshness': { + 'cached': _cache['timestamp'] is not None, + 'cache_age_seconds': (datetime.now() - _cache['timestamp']).seconds if _cache['timestamp'] else None + } + }) + +@app.errorhandler(404) +def not_found(error): + return jsonify({'error': 'Endpoint not found'}), 404 + +@app.errorhandler(500) +def internal_error(error): + return jsonify({'error': 'Internal server error'}), 500 + +if __name__ == '__main__': + print("🚀 Starting Crypto Ecosystems API...") + print("📡 API will be available at: http://localhost:5000") + print("📚 Documentation: http://localhost:5000/") + app.run(debug=True, host='0.0.0.0', port=5000) \ No newline at end of file diff --git a/api/requirements.txt b/api/requirements.txt new file mode 100644 index 0000000000..60763e5bbc --- /dev/null +++ b/api/requirements.txt @@ -0,0 +1,2 @@ +Flask==3.0.0 +Flask-CORS==4.0.0 \ No newline at end of file diff --git a/migrations/2025-10-05T220000_add_algorand b/migrations/2025-10-05T220000_add_algorand new file mode 100644 index 0000000000..d08d6dcda3 --- /dev/null +++ b/migrations/2025-10-05T220000_add_algorand @@ -0,0 +1,7 @@ +-- Add Algorand ecosystem and key repos +ecoadd Algorand +repadd Algorand https://github.com/algorand/go-algorand #protocol +repadd Algorand https://github.com/algorand/js-algorand-sdk #sdk +repadd Algorand https://github.com/algorand/py-algorand-sdk #sdk +repadd Algorand https://github.com/algorand/go-algorand-sdk #sdk +repadd Algorand https://github.com/algorand/go-algorand-doc #docs