From a35a1b4362bfedc558866653413134e7ad1ec381 Mon Sep 17 00:00:00 2001 From: armorpro Date: Fri, 3 Oct 2025 15:57:26 +0300 Subject: [PATCH 1/6] Fix typo: Lighting -> Lightning in README --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index c19f437409..851060113f 100644 --- a/README.md +++ b/README.md @@ -56,8 +56,7 @@ ecoadd Lightning repadd Lightning https://github.com/lightningnetwork/lnd #protocol -- Connect ecosystems using the ecocon keyword. -- The following connects Lighting as a sub ecosystem of Bitcoin. -ecocon Bitcoin Lighting -``` +ecocon Bitcoin Lightning``` ## How to Give Attribution For Usage of the Electric Capital Crypto Ecosystems From 8a34caafbec806ded82cdca24e9d820f2d430fc9 Mon Sep 17 00:00:00 2001 From: armorpro Date: Fri, 3 Oct 2025 15:57:55 +0300 Subject: [PATCH 2/6] Add comprehensive contribution guide --- CONTRIBUTING.md | 476 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 476 insertions(+) create mode 100644 CONTRIBUTING.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000000..70ddec72e6 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,476 @@ +\# Contributing to Crypto Ecosystems + + + +Thank you for your interest in contributing to Crypto Ecosystems! 🎉 + + + +This guide will help you add new ecosystems, repositories, and make improvements to the project. + + + +\## Table of Contents + +\- \[Getting Started](#getting-started) + +\- \[Adding New Ecosystems](#adding-new-ecosystems) + +\- \[Adding Repositories](#adding-repositories) + +\- \[Connecting Ecosystems](#connecting-ecosystems) + +\- \[Migration File Format](#migration-file-format) + +\- \[Testing Your Changes](#testing-your-changes) + +\- \[Submitting Your Contribution](#submitting-your-contribution) + + + +\## Getting Started + + + +1\. \*\*Fork the repository\*\* to your GitHub account + +2\. \*\*Clone your fork\*\* locally: + +  ```bash + +  git clone https://github.com/YOUR\_USERNAME/crypto-ecosystems.git + +  cd crypto-ecosystems + +  ``` + +3\. \*\*Create a new branch\*\* for your changes: + +  ```bash + +  git checkout -b add-new-ecosystem + +  ``` + + + +\## Adding New Ecosystems + + + +To add a new ecosystem, create a migration file in the `migrations/` directory. + + + +\### Migration File Naming Convention + +``` + +migrations/YYYY-MM-DDThhmmss\_description\_of\_your\_migration + +``` + + + +\*\*Example:\*\* + +``` + +migrations/2025-10-03T153000\_add\_solana\_ecosystem + +``` + + + +\### Migration File Content + + + +Use the Domain Specific Language (DSL) keywords: + + + +\#### 1. \*\*Add Ecosystem\*\* - `ecoadd` + +``` + +-- Add a new ecosystem + +ecoadd Solana + +``` + + + +\#### 2. \*\*Add Repository\*\* - `repadd` + +``` + +-- Add repositories to the ecosystem + +repadd Solana https://github.com/solana-labs/solana #protocol + +repadd Solana https://github.com/project-serum/serum-dex #defi + +``` + + + +\*\*Available Tags:\*\* + +\- `#protocol` - Core protocol repositories + +\- `#defi` - Decentralized Finance + +\- `#nft` - Non-Fungible Tokens + +\- `#wallet` - Wallet implementations + +\- `#developer-tool` - Development tools + +\- `#bridge` - Cross-chain bridges + +\- `#oracle` - Oracle services + +\- `#dao` - Decentralized Autonomous Organizations + + + +\#### 3. \*\*Connect Ecosystems\*\* - `ecocon` + +``` + +-- Connect sub-ecosystems to parent ecosystems + +ecocon Ethereum Uniswap + +ecocon Bitcoin Lightning + +``` + + + +\### Complete Example Migration File + + + +\*\*File:\*\* `migrations/2025-10-03T153000\_add\_solana\_ecosystem` + + + +``` + +-- Add Solana ecosystem + +ecoadd Solana + + + +-- Add core protocol repos + +repadd Solana https://github.com/solana-labs/solana #protocol + +repadd Solana https://github.com/solana-labs/solana-program-library #protocol + + + +-- Add DeFi projects + +repadd Solana https://github.com/project-serum/serum-dex #defi + +repadd Solana https://github.com/solana-labs/solana-web3.js #developer-tool + + + +-- Add sub-ecosystem + +ecoadd Serum + +repadd Serum https://github.com/project-serum/serum-ts #developer-tool + +ecocon Solana Serum + +``` + + + +\## Adding Repositories + + + +If you want to add repositories to an \*\*existing ecosystem\*\*, simply create a migration file with `repadd` commands: + + + +\*\*File:\*\* `migrations/2025-10-03T154500\_add\_ethereum\_repos` + + + +``` + +-- Add new Ethereum repositories + +repadd Ethereum https://github.com/ethereum/go-ethereum #protocol + +repadd Ethereum https://github.com/OpenZeppelin/openzeppelin-contracts #developer-tool + +repadd Ethereum https://github.com/Uniswap/v3-core #defi + +``` + + + +\## Connecting Ecosystems + + + +To establish parent-child relationships between ecosystems: + + + +``` + +-- Polygon is a sub-ecosystem of Ethereum + +ecoadd Polygon + +ecocon Ethereum Polygon + + + +-- Optimism is a Layer 2 of Ethereum + +ecoadd Optimism + +ecocon Ethereum Optimism + +``` + + + +\## Migration File Format + + + +\### Rules: + +1\. \*\*Comments\*\* start with `--` + +2\. \*\*One command per line\*\* + +3\. \*\*Datetime format:\*\* `YYYY-MM-DDThhmmss` (ISO 8601 without colons) + +4\. \*\*No special characters\*\* in filenames except underscore `\_` + +5\. \*\*Descriptive names\*\* for clarity + + + +\### Good Examples: + +``` + +migrations/2025-10-03T120000\_add\_bitcoin\_lightning + +migrations/2025-10-03T130000\_add\_defi\_protocols + +migrations/2025-10-03T140000\_connect\_layer2\_networks + +``` + + + +\### Bad Examples: + +``` + +migrations/2025-10-03\_new\_stuff + +migrations/add\_repos + +migrations/2025:10:03T12:00:00\_update + +``` + + + +\## Testing Your Changes + + + +Before submitting, test your migration: + + + +\### On Linux/Mac: + +```bash + +./run.sh export test\_output.jsonl + +``` + + + +\### On Windows: + +```bash + +\# Install Zig if not already installed + +\# Then run: + +zig build run -- export test\_output.jsonl + +``` + + + +Check if your changes appear in the output: + +```bash + +\# On Linux/Mac + +cat test\_output.jsonl | grep "your\_ecosystem\_name" + + + +\# On Windows + +findstr "your\_ecosystem\_name" test\_output.jsonl + +``` + + + +\## Submitting Your Contribution + + + +1\. \*\*Commit your changes:\*\* + +  ```bash + +  git add migrations/ + +  git commit -m "Add Solana ecosystem with core repositories" + +  ``` + + + +2\. \*\*Push to your fork:\*\* + +  ```bash + +  git push origin add-new-ecosystem + +  ``` + + + +3\. \*\*Create a Pull Request:\*\* + +  - Go to the original repository on GitHub + +  - Click "New Pull Request" + +  - Select your branch + +  - Fill in the PR template with: + +  - \*\*Description:\*\* What ecosystems/repos you added + +  - \*\*Motivation:\*\* Why they should be included + +  - \*\*Testing:\*\* Confirm you tested the migration + + + +4\. \*\*PR Title Format:\*\* + +  ``` + +  Add \[Ecosystem Name] ecosystem + +  Add repositories to \[Ecosystem Name] + +  Connect \[Child] to \[Parent] ecosystem + +  ``` + + + +\## Quality Guidelines + + + +\### Repository Requirements: + +\- ✅ Must be \*\*open source\*\* + +\- ✅ Must be related to \*\*blockchain/crypto/web3\*\* + +\- ✅ Must be \*\*actively maintained\*\* (or historically significant) + +\- ✅ Must have a \*\*valid GitHub URL\*\* + +\- ❌ No private repositories + +\- ❌ No spam or low-quality projects + + + +\### Ecosystem Requirements: + +\- ✅ Must be a \*\*recognized blockchain ecosystem\*\* + +\- ✅ Must have multiple \*\*active repositories\*\* + +\- ✅ Must be \*\*publicly documented\*\* + + + +\## Need Help? + + + +\- 📖 Check the \[README.md](README.md) for usage examples + +\- 🐛 Report bugs in \[Issues](https://github.com/PROFADAM/crypto-ecosystems/issues) + +\- 💬 Ask questions in \[Discussions](https://github.com/PROFADAM/crypto-ecosystems/discussions) + +\- 📧 Contact: \[Your Contact Info] + + + +\## Code of Conduct + + + +\- Be respectful and constructive + +\- Focus on the technology, not personal opinions + +\- Provide accurate information + +\- Help others learn and contribute + + + +\## License + + + +By contributing, you agree that your contributions will be licensed under the MIT License. + + + +--- + + + +Thank you for helping build the most comprehensive crypto ecosystem taxonomy! 🚀 + From cad72b7bd7a3de04ba1b75b59e5b5d329370de94 Mon Sep 17 00:00:00 2001 From: armorpro Date: Fri, 3 Oct 2025 15:59:46 +0300 Subject: [PATCH 3/6] Add CI/CD workflows: validation and auto-export --- .github/workflows/export-data.yml | 181 ++++++++++++++++++++++++ .github/workflows/validate.yml | 222 ++++++++++++++++++++++++++++++ 2 files changed, 403 insertions(+) create mode 100644 .github/workflows/export-data.yml create mode 100644 .github/workflows/validate.yml diff --git a/.github/workflows/export-data.yml b/.github/workflows/export-data.yml new file mode 100644 index 0000000000..e3d6f89fd1 --- /dev/null +++ b/.github/workflows/export-data.yml @@ -0,0 +1,181 @@ +name: Export Ecosystem Data + +on: + # Run weekly on Sundays at midnight + schedule: + - cron: '0 0 * * 0' + # Allow manual trigger + workflow_dispatch: + # Run on pushes to master + push: + branches: [ master, main ] + paths: + - 'migrations/**' + +jobs: + export-all: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Zig + uses: goto-bus-stop/setup-zig@v2 + with: + version: 0.11.0 + + - name: Build project + run: zig build + + - name: Get current date + id: date + run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT + + - name: Export all ecosystems + run: | + ./run.sh export exports/crypto-ecosystems-full-${{ steps.date.outputs.date }}.jsonl + echo "✅ Full export completed" + + - name: Generate ecosystem list + run: | + jq -r '.eco_name' exports/crypto-ecosystems-full-${{ steps.date.outputs.date }}.jsonl | \ + sort -u > exports/ecosystem-list-${{ steps.date.outputs.date }}.txt + echo "✅ Ecosystem list generated" + + - name: Export popular ecosystems individually + run: | + mkdir -p exports/individual + + # List of major ecosystems to export separately + ecosystems=("Bitcoin" "Ethereum" "Solana" "Polkadot" "Cardano" "Avalanche" "Polygon") + + for eco in "${ecosystems[@]}"; do + if [ -d "data/ecosystems/$eco" ] || grep -q "^ecoadd $eco$" migrations/*; then + echo "Exporting $eco..." + ./run.sh export -e "$eco" "exports/individual/${eco,,}-${{ steps.date.outputs.date }}.jsonl" + fi + done + + echo "✅ Individual exports completed" + + - name: Generate statistics JSON + run: | + cat > exports/statistics-${{ steps.date.outputs.date }}.json < exports/README.md < line.trim()) + .map(line => JSON.parse(line)); + \`\`\` + + ## Format + + Each line is a JSON object with the following structure: + \`\`\`json + { + "eco_name": "Bitcoin", + "branch": ["Lightning"], + "repo_url": "https://github.com/lightningnetwork/lnd", + "tags": ["#protocol"] + } + \`\`\` + + ## License + + This data is licensed under MIT License. See LICENSE file for details. + EOF + + echo "✅ Export README generated" + + - name: Create release archive + run: | + cd exports + tar -czf crypto-ecosystems-${{ steps.date.outputs.date }}.tar.gz *.jsonl *.txt *.json *.md individual/ + cd .. + echo "✅ Archive created" + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: crypto-ecosystems-export-${{ steps.date.outputs.date }} + path: | + exports/*.jsonl + exports/*.txt + exports/*.json + exports/*.md + exports/individual/ + retention-days: 90 + + - name: Upload release archive + uses: actions/upload-artifact@v4 + with: + name: crypto-ecosystems-archive-${{ steps.date.outputs.date }} + path: exports/*.tar.gz + retention-days: 365 + + - name: Create release (on schedule or manual trigger) + if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' + uses: softprops/action-gh-release@v1 + with: + tag_name: data-export-${{ steps.date.outputs.date }} + name: Data Export ${{ steps.date.outputs.date }} + body: | + Automated weekly export of crypto ecosystem data. + + **Statistics:** + - Total Repositories: $(jq -s 'length' exports/crypto-ecosystems-full-${{ steps.date.outputs.date }}.jsonl) + - Total Ecosystems: $(jq -r '.eco_name' exports/crypto-ecosystems-full-${{ steps.date.outputs.date }}.jsonl | sort -u | wc -l) + + Download the complete archive or individual files below. + files: | + exports/crypto-ecosystems-${{ steps.date.outputs.date }}.tar.gz + exports/crypto-ecosystems-full-${{ steps.date.outputs.date }}.jsonl + exports/statistics-${{ steps.date.outputs.date }}.json + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml new file mode 100644 index 0000000000..aa0e787034 --- /dev/null +++ b/.github/workflows/validate.yml @@ -0,0 +1,222 @@ +name: Validate Crypto Ecosystems + +on: + push: + branches: [ master, main ] + pull_request: + branches: [ master, main ] + schedule: + # Run daily at 2 AM UTC to check for broken links + - cron: '0 2 * * *' + +jobs: + validate-migrations: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Setup Zig + uses: goto-bus-stop/setup-zig@v2 + with: + version: 0.11.0 + + - name: Build project + run: zig build + + - name: Run tests + run: zig build test + + - name: Validate migration files + run: | + echo "Checking migration file naming convention..." + for file in migrations/*; do + if [[ -f "$file" ]]; then + basename=$(basename "$file") + # Check if filename matches YYYY-MM-DDThhmmss_description format + if [[ ! "$basename" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{6}_.+ ]]; then + echo "❌ Invalid migration filename: $basename" + echo "Expected format: YYYY-MM-DDThhmmss_description" + exit 1 + fi + fi + done + echo "✅ All migration files follow naming convention" + + - name: Export ecosystems + run: | + ./run.sh export test_output.jsonl + if [ ! -f test_output.jsonl ]; then + echo "❌ Export failed - no output file generated" + exit 1 + fi + echo "✅ Export successful" + + - name: Validate JSON output + run: | + echo "Validating JSONL output..." + line_count=0 + while IFS= read -r line; do + line_count=$((line_count + 1)) + if ! echo "$line" | jq empty 2>/dev/null; then + echo "❌ Invalid JSON at line $line_count: $line" + exit 1 + fi + done < test_output.jsonl + echo "✅ All $line_count lines are valid JSON" + + - name: Check for duplicate repositories + run: | + echo "Checking for duplicate repository URLs..." + duplicates=$(jq -r '.repo_url' test_output.jsonl | sort | uniq -d) + if [ ! -z "$duplicates" ]; then + echo "❌ Duplicate repositories found:" + echo "$duplicates" + exit 1 + fi + echo "✅ No duplicate repositories found" + + - name: Generate statistics + run: | + echo "📊 Ecosystem Statistics:" + echo "========================" + total_repos=$(jq -s 'length' test_output.jsonl) + total_ecosystems=$(jq -r '.eco_name' test_output.jsonl | sort -u | wc -l) + echo "Total Ecosystems: $total_ecosystems" + echo "Total Repositories: $total_repos" + echo "" + echo "Top 10 Ecosystems by Repository Count:" + jq -r '.eco_name' test_output.jsonl | sort | uniq -c | sort -rn | head -10 + + - name: Upload export artifact + uses: actions/upload-artifact@v4 + with: + name: ecosystem-export + path: test_output.jsonl + retention-days: 30 + + check-links: + runs-on: ubuntu-latest + if: github.event_name == 'schedule' || github.event_name == 'pull_request' + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Zig + uses: goto-bus-stop/setup-zig@v2 + with: + version: 0.11.0 + + - name: Build and export + run: | + zig build + ./run.sh export repos.jsonl + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + + - name: Check repository accessibility + run: | + echo "Checking repository URLs..." + broken_links=() + checked=0 + + while IFS= read -r line; do + url=$(echo "$line" | jq -r '.repo_url') + checked=$((checked + 1)) + + # Rate limit: check every 2 seconds + if [ $((checked % 10)) -eq 0 ]; then + echo "Checked $checked repositories..." + sleep 2 + fi + + # Check if URL is accessible (HTTP 200 or 301/302 redirect) + status=$(curl -o /dev/null -s -w "%{http_code}" -L "$url" --max-time 10) + + if [ "$status" != "200" ] && [ "$status" != "301" ] && [ "$status" != "302" ]; then + broken_links+=("$url (HTTP $status)") + fi + done < repos.jsonl + + if [ ${#broken_links[@]} -gt 0 ]; then + echo "⚠️ Found ${#broken_links[@]} potentially broken links:" + printf '%s\n' "${broken_links[@]}" + # Don't fail the build, just warn + exit 0 + else + echo "✅ All $checked repository URLs are accessible" + fi + + lint-migrations: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Check migration syntax + run: | + echo "Linting migration files..." + errors=0 + + for file in migrations/*; do + if [[ -f "$file" ]]; then + echo "Checking $file..." + + # Check for valid keywords + while IFS= read -r line; do + # Skip comments and empty lines + if [[ "$line" =~ ^[[:space:]]*-- ]] || [[ -z "$line" ]]; then + continue + fi + + # Check if line starts with valid keyword + if [[ ! "$line" =~ ^(ecoadd|repadd|ecocon)[[:space:]] ]]; then + echo "❌ Invalid syntax in $file: $line" + errors=$((errors + 1)) + fi + + # Validate repadd format (keyword ecosystem url [tag]) + if [[ "$line" =~ ^repadd[[:space:]] ]]; then + if [[ ! "$line" =~ ^repadd[[:space:]]+[^[:space:]]+[[:space:]]+https?:// ]]; then + echo "❌ Invalid repadd format in $file: $line" + errors=$((errors + 1)) + fi + fi + done < "$file" + fi + done + + if [ $errors -gt 0 ]; then + echo "❌ Found $errors syntax errors" + exit 1 + fi + echo "✅ All migration files have valid syntax" + + security-scan: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Run Trivy security scanner + uses: aquasecurity/trivy-action@master + with: + scan-type: 'fs' + scan-ref: '.' + format: 'sarif' + output: 'trivy-results.sarif' + + - name: Upload Trivy results to GitHub Security + uses: github/codeql-action/upload-sarif@v3 + if: always() + with: + sarif_file: 'trivy-results.sarif' \ No newline at end of file From 726faa08bda549d16748cf88da043eee4a9602b4 Mon Sep 17 00:00:00 2001 From: armorpro Date: Fri, 3 Oct 2025 16:02:00 +0300 Subject: [PATCH 4/6] Add REST API with Flask: endpoints for ecosystems, repos, search, and stats --- api/README.md | 622 +++++++++++++++++++++++++++++++++++++++++++ api/app.py | 272 +++++++++++++++++++ api/requirements.txt | 2 + 3 files changed, 896 insertions(+) create mode 100644 api/README.md create mode 100644 api/app.py create mode 100644 api/requirements.txt diff --git a/api/README.md b/api/README.md new file mode 100644 index 0000000000..7ef1659aca --- /dev/null +++ b/api/README.md @@ -0,0 +1,622 @@ +\# Crypto Ecosystems REST API + + + +A Flask-based REST API for accessing crypto ecosystem taxonomy data. + + + +\## Installation + + + +\### Prerequisites + +\- Python 3.8 or higher + +\- Zig (for building the main project) + + + +\### Setup + + + +1\. \*\*Install Python dependencies:\*\* + +  ```bash + +  cd api + +  pip install -r requirements.txt + +  ``` + + + +2\. \*\*Build the main project (required for data export):\*\* + +  ```bash + +  cd .. + +  zig build + +  ``` + + + +\## Running the API + + + +\### Development Mode + +```bash + +cd api + +python app.py + +``` + + + +The API will start at `http://localhost:5000` + + + +\### Production Mode (with Gunicorn) + +```bash + +pip install gunicorn + +gunicorn -w 4 -b 0.0.0.0:5000 app:app + +``` + + + +\## API Endpoints + + + +\### 📚 Documentation + +``` + +GET / + +``` + +Returns API documentation and available endpoints. + + + +\### 🌐 Get All Ecosystems + +``` + +GET /api/ecosystems + +``` + +Returns a list of all ecosystems with repository counts. + + + +\*\*Response:\*\* + +```json + +{ + +  "total": 150, + +  "ecosystems": \[ + +  { + +  "name": "Bitcoin", + +  "repository\_count": 523, + +  "sub\_ecosystem\_count": 5, + +  "sub\_ecosystems": \["Lightning", "Liquid"] + +  } + +  ] + +} + +``` + + + +\### 🔍 Get Specific Ecosystem + +``` + +GET /api/ecosystems/ + +``` + +Returns detailed information about a specific ecosystem. + + + +\*\*Example:\*\* + +```bash + +curl http://localhost:5000/api/ecosystems/Bitcoin + +``` + + + +\*\*Response:\*\* + +```json + +{ + +  "name": "Bitcoin", + +  "repository\_count": 523, + +  "repositories": \["https://github.com/bitcoin/bitcoin", ...], + +  "sub\_ecosystems": \["Lightning", "Liquid"], + +  "tags": { + +  "#protocol": 45, + +  "#wallet": 32, + +  "#developer-tool": 28 + +  }, + +  "sample\_repos": \[...] + +} + +``` + + + +\### 📦 Get All Repositories + +``` + +GET /api/repositories?page=1\&per\_page=50 + +``` + +Returns paginated list of all repositories. + + + +\*\*Query Parameters:\*\* + +\- `page` (default: 1) - Page number + +\- `per\_page` (default: 50, max: 200) - Items per page + + + +\*\*Response:\*\* + +```json + +{ + +  "total": 15234, + +  "page": 1, + +  "per\_page": 50, + +  "total\_pages": 305, + +  "repositories": \[...] + +} + +``` + + + +\### 🔎 Search Repositories + +``` + +GET /api/repositories/search?q= + +``` + +Search repositories by name, ecosystem, or tags. + + + +\*\*Example:\*\* + +```bash + +curl http://localhost:5000/api/repositories/search?q=ethereum + +``` + + + +\*\*Response:\*\* + +```json + +{ + +  "query": "ethereum", + +  "total": 234, + +  "results": \[...] + +} + +``` + + + +\### 🏷️ Get All Tags + +``` + +GET /api/tags + +``` + +Returns all available tags with their usage counts. + + + +\*\*Response:\*\* + +```json + +{ + +  "total\_tags": 15, + +  "tags": \[ + +  {"tag": "#protocol", "count": 423}, + +  {"tag": "#defi", "count": 312}, + +  {"tag": "#wallet", "count": 245} + +  ] + +} + +``` + + + +\### 📊 Get Statistics + +``` + +GET /api/stats + +``` + +Returns overall statistics about the dataset. + + + +\*\*Response:\*\* + +```json + +{ + +  "timestamp": "2025-10-03T16:30:00", + +  "total\_ecosystems": 156, + +  "total\_repositories": 15234, + +  "total\_tags": 15, + +  "top\_ecosystems": \[ + +  {"name": "Ethereum", "repository\_count": 3245}, + +  {"name": "Bitcoin", "repository\_count": 1523} + +  ], + +  "tag\_distribution": \[...], + +  "data\_freshness": { + +  "cached": true, + +  "cache\_age\_seconds": 245 + +  } + +} + +``` + + + +\### ❤️ Health Check + +``` + +GET /api/health + +``` + +Returns API health status. + + + +\*\*Response:\*\* + +```json + +{ + +  "status": "healthy", + +  "timestamp": "2025-10-03T16:30:00", + +  "cache\_age\_seconds": 245 + +} + +``` + + + +\## Usage Examples + + + +\### Python + +```python + +import requests + + + +\# Get all ecosystems + +response = requests.get('http://localhost:5000/api/ecosystems') + +ecosystems = response.json() + + + +\# Search for Ethereum repositories + +response = requests.get('http://localhost:5000/api/repositories/search?q=ethereum') + +results = response.json() + + + +\# Get statistics + +response = requests.get('http://localhost:5000/api/stats') + +stats = response.json() + +print(f"Total repos: {stats\['total\_repositories']}") + +``` + + + +\### JavaScript (Node.js) + +```javascript + +const axios = require('axios'); + + + +async function getEcosystems() { + +  const response = await axios.get('http://localhost:5000/api/ecosystems'); + +  console.log(response.data); + +} + + + +async function searchRepos(query) { + +  const response = await axios.get( + +  `http://localhost:5000/api/repositories/search?q=${query}` + +  ); + +  return response.data.results; + +} + +``` + + + +\### cURL + +```bash + +\# Get all ecosystems + +curl http://localhost:5000/api/ecosystems + + + +\# Get Bitcoin ecosystem details + +curl http://localhost:5000/api/ecosystems/Bitcoin + + + +\# Search for DeFi projects + +curl "http://localhost:5000/api/repositories/search?q=defi" + + + +\# Get statistics + +curl http://localhost:5000/api/stats + +``` + + + +\## Caching + + + +The API caches exported data for 1 hour (3600 seconds) to improve performance. After the cache expires, data is automatically refreshed from the latest migrations. + + + +\## CORS + + + +CORS is enabled for all origins, making the API accessible from web applications. + + + +\## Error Handling + + + +The API returns standard HTTP status codes: + +\- `200` - Success + +\- `400` - Bad Request (missing or invalid parameters) + +\- `404` - Not Found (ecosystem or endpoint doesn't exist) + +\- `500` - Internal Server Error + + + +Error responses follow this format: + +```json + +{ + +  "error": "Error description" + +} + +``` + + + +\## Docker Deployment + + + +Create a `Dockerfile`: + +```dockerfile + +FROM python:3.11-slim + + + +WORKDIR /app + + + +\# Install Zig + +RUN apt-get update \&\& apt-get install -y wget xz-utils + +RUN wget https://ziglang.org/download/0.11.0/zig-linux-x86\_64-0.11.0.tar.xz + +RUN tar -xf zig-linux-x86\_64-0.11.0.tar.xz + +ENV PATH="/app/zig-linux-x86\_64-0.11.0:${PATH}" + + + +\# Copy project files + +COPY . . + + + +\# Build project + +RUN zig build + + + +\# Install Python dependencies + +RUN pip install -r api/requirements.txt + + + +EXPOSE 5000 + + + +CMD \["python", "api/app.py"] + +``` + + + +Build and run: + +```bash + +docker build -t crypto-ecosystems-api . + +docker run -p 5000:5000 crypto-ecosystems-api + +``` + + + +\## Contributing + + + +See \[CONTRIBUTING.md](../CONTRIBUTING.md) for guidelines on contributing to this project. + + + +\## License + + + +MIT License - See \[LICENSE](../LICENSE) for details. + diff --git a/api/app.py b/api/app.py new file mode 100644 index 0000000000..f829344a10 --- /dev/null +++ b/api/app.py @@ -0,0 +1,272 @@ +from flask import Flask, jsonify, request +from flask_cors import CORS +import json +import os +import subprocess +from collections import defaultdict, Counter +from datetime import datetime + +app = Flask(__name__) +CORS(app) # Enable CORS for all routes + +# Cache for exported data +_cache = { + 'data': None, + 'timestamp': None, + 'ttl': 3600 # Cache for 1 hour +} + +def load_ecosystem_data(): + """Load ecosystem data from export or cache""" + current_time = datetime.now() + + # Return cached data if still valid + if _cache['data'] and _cache['timestamp']: + time_diff = (current_time - _cache['timestamp']).seconds + if time_diff < _cache['ttl']: + return _cache['data'] + + # Generate fresh export + export_file = 'api_export.jsonl' + + try: + # Run export command + if os.name == 'nt': # Windows + subprocess.run(['cmd', '/c', 'run.sh', 'export', export_file], check=True) + else: # Linux/Mac + subprocess.run(['./run.sh', 'export', export_file], check=True) + + # Load data + data = [] + with open(export_file, 'r', encoding='utf-8') as f: + for line in f: + if line.strip(): + data.append(json.loads(line)) + + # Update cache + _cache['data'] = data + _cache['timestamp'] = current_time + + # Clean up export file + os.remove(export_file) + + return data + except Exception as e: + print(f"Error loading data: {e}") + return _cache['data'] if _cache['data'] else [] + +@app.route('/') +def index(): + """API documentation""" + return jsonify({ + 'name': 'Crypto Ecosystems API', + 'version': '1.0.0', + 'description': 'REST API for crypto ecosystem taxonomy data', + 'endpoints': { + '/': 'API documentation (this page)', + '/api/ecosystems': 'List all ecosystems', + '/api/ecosystems/': 'Get specific ecosystem details', + '/api/repositories': 'List all repositories', + '/api/repositories/search?q=': 'Search repositories', + '/api/stats': 'Get overall statistics', + '/api/tags': 'List all available tags', + '/api/health': 'API health check' + }, + 'documentation': 'https://github.com/PROFADAM/crypto-ecosystems' + }) + +@app.route('/api/health') +def health(): + """Health check endpoint""" + return jsonify({ + 'status': 'healthy', + 'timestamp': datetime.now().isoformat(), + 'cache_age_seconds': (datetime.now() - _cache['timestamp']).seconds if _cache['timestamp'] else None + }) + +@app.route('/api/ecosystems') +def get_ecosystems(): + """Get list of all ecosystems with their repository counts""" + data = load_ecosystem_data() + + # Group by ecosystem + ecosystems = defaultdict(lambda: {'repos': [], 'sub_ecosystems': set()}) + + for item in data: + eco_name = item['eco_name'] + ecosystems[eco_name]['repos'].append(item['repo_url']) + + # Track sub-ecosystems + if item.get('branch'): + for branch in item['branch']: + ecosystems[eco_name]['sub_ecosystems'].add(branch) + + # Format response + result = [] + for name, info in sorted(ecosystems.items()): + result.append({ + 'name': name, + 'repository_count': len(info['repos']), + 'sub_ecosystem_count': len(info['sub_ecosystems']), + 'sub_ecosystems': sorted(list(info['sub_ecosystems'])) + }) + + return jsonify({ + 'total': len(result), + 'ecosystems': result + }) + +@app.route('/api/ecosystems/') +def get_ecosystem(ecosystem_name): + """Get detailed information about a specific ecosystem""" + data = load_ecosystem_data() + + # Filter by ecosystem name (case-insensitive) + ecosystem_data = [ + item for item in data + if item['eco_name'].lower() == ecosystem_name.lower() + ] + + if not ecosystem_data: + return jsonify({'error': 'Ecosystem not found'}), 404 + + # Collect statistics + repos = [item['repo_url'] for item in ecosystem_data] + tags = [] + for item in ecosystem_data: + tags.extend(item.get('tags', [])) + + sub_ecosystems = set() + for item in ecosystem_data: + if item.get('branch'): + sub_ecosystems.update(item['branch']) + + return jsonify({ + 'name': ecosystem_data[0]['eco_name'], + 'repository_count': len(repos), + 'repositories': repos, + 'sub_ecosystems': sorted(list(sub_ecosystems)), + 'tags': dict(Counter(tags)), + 'sample_repos': repos[:10] # First 10 repos as sample + }) + +@app.route('/api/repositories') +def get_repositories(): + """Get all repositories with pagination""" + data = load_ecosystem_data() + + # Pagination + page = request.args.get('page', 1, type=int) + per_page = request.args.get('per_page', 50, type=int) + per_page = min(per_page, 200) # Max 200 per page + + start = (page - 1) * per_page + end = start + per_page + + total = len(data) + repos = data[start:end] + + return jsonify({ + 'total': total, + 'page': page, + 'per_page': per_page, + 'total_pages': (total + per_page - 1) // per_page, + 'repositories': repos + }) + +@app.route('/api/repositories/search') +def search_repositories(): + """Search repositories by query""" + query = request.args.get('q', '').lower() + + if not query: + return jsonify({'error': 'Query parameter "q" is required'}), 400 + + data = load_ecosystem_data() + + # Search in repo URL and ecosystem name + results = [ + item for item in data + if query in item['repo_url'].lower() or + query in item['eco_name'].lower() or + any(query in tag.lower() for tag in item.get('tags', [])) + ] + + return jsonify({ + 'query': query, + 'total': len(results), + 'results': results + }) + +@app.route('/api/tags') +def get_tags(): + """Get all available tags with their counts""" + data = load_ecosystem_data() + + tags = [] + for item in data: + tags.extend(item.get('tags', [])) + + tag_counts = Counter(tags) + + return jsonify({ + 'total_tags': len(tag_counts), + 'tags': [ + {'tag': tag, 'count': count} + for tag, count in tag_counts.most_common() + ] + }) + +@app.route('/api/stats') +def get_stats(): + """Get overall statistics""" + data = load_ecosystem_data() + + # Calculate statistics + ecosystems = set(item['eco_name'] for item in data) + repos = set(item['repo_url'] for item in data) + + tags = [] + for item in data: + tags.extend(item.get('tags', [])) + + # Top ecosystems by repo count + eco_counts = Counter(item['eco_name'] for item in data) + top_ecosystems = [ + {'name': name, 'repository_count': count} + for name, count in eco_counts.most_common(10) + ] + + # Tag distribution + tag_counts = Counter(tags) + tag_distribution = [ + {'tag': tag, 'count': count} + for tag, count in tag_counts.most_common(10) + ] + + return jsonify({ + 'timestamp': datetime.now().isoformat(), + 'total_ecosystems': len(ecosystems), + 'total_repositories': len(repos), + 'total_tags': len(set(tags)), + 'top_ecosystems': top_ecosystems, + 'tag_distribution': tag_distribution, + 'data_freshness': { + 'cached': _cache['timestamp'] is not None, + 'cache_age_seconds': (datetime.now() - _cache['timestamp']).seconds if _cache['timestamp'] else None + } + }) + +@app.errorhandler(404) +def not_found(error): + return jsonify({'error': 'Endpoint not found'}), 404 + +@app.errorhandler(500) +def internal_error(error): + return jsonify({'error': 'Internal server error'}), 500 + +if __name__ == '__main__': + print("🚀 Starting Crypto Ecosystems API...") + print("📡 API will be available at: http://localhost:5000") + print("📚 Documentation: http://localhost:5000/") + app.run(debug=True, host='0.0.0.0', port=5000) \ No newline at end of file diff --git a/api/requirements.txt b/api/requirements.txt new file mode 100644 index 0000000000..60763e5bbc --- /dev/null +++ b/api/requirements.txt @@ -0,0 +1,2 @@ +Flask==3.0.0 +Flask-CORS==4.0.0 \ No newline at end of file From 846a55988320e99dfbcc02d5cf565400b15f539b Mon Sep 17 00:00:00 2001 From: PROFADAM Date: Sun, 5 Oct 2025 23:42:08 +0300 Subject: [PATCH 5/6] Add algokit-client-generator-ts to Algorand ecosystem --- migrations/2025-10-05T112301_add_algorand_client_generator_ts | 1 + 1 file changed, 1 insertion(+) create mode 100644 migrations/2025-10-05T112301_add_algorand_client_generator_ts diff --git a/migrations/2025-10-05T112301_add_algorand_client_generator_ts b/migrations/2025-10-05T112301_add_algorand_client_generator_ts new file mode 100644 index 0000000000..a6c9f6016a --- /dev/null +++ b/migrations/2025-10-05T112301_add_algorand_client_generator_ts @@ -0,0 +1 @@ +repadd Algorand https://github.com/algorandfoundation/algokit-client-generator-ts #generator #typescript From 16a7d9bb4f4b58464545ee954b94c2c52cc4665b Mon Sep 17 00:00:00 2001 From: PROFADAM Date: Mon, 6 Oct 2025 00:03:20 +0300 Subject: [PATCH 6/6] revert: make PR data-only; remove unintended files, keep only migration --- .github/workflows/export-data.yml | 181 --------- .github/workflows/validate.yml | 222 ----------- CONTRIBUTING.md | 476 ----------------------- README.md | 3 +- api/README.md | 622 ------------------------------ api/app.py | 272 ------------- api/requirements.txt | 2 - 7 files changed, 2 insertions(+), 1776 deletions(-) delete mode 100644 .github/workflows/export-data.yml delete mode 100644 .github/workflows/validate.yml delete mode 100644 CONTRIBUTING.md delete mode 100644 api/README.md delete mode 100644 api/app.py delete mode 100644 api/requirements.txt diff --git a/.github/workflows/export-data.yml b/.github/workflows/export-data.yml deleted file mode 100644 index e3d6f89fd1..0000000000 --- a/.github/workflows/export-data.yml +++ /dev/null @@ -1,181 +0,0 @@ -name: Export Ecosystem Data - -on: - # Run weekly on Sundays at midnight - schedule: - - cron: '0 0 * * 0' - # Allow manual trigger - workflow_dispatch: - # Run on pushes to master - push: - branches: [ master, main ] - paths: - - 'migrations/**' - -jobs: - export-all: - runs-on: ubuntu-latest - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Setup Zig - uses: goto-bus-stop/setup-zig@v2 - with: - version: 0.11.0 - - - name: Build project - run: zig build - - - name: Get current date - id: date - run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT - - - name: Export all ecosystems - run: | - ./run.sh export exports/crypto-ecosystems-full-${{ steps.date.outputs.date }}.jsonl - echo "✅ Full export completed" - - - name: Generate ecosystem list - run: | - jq -r '.eco_name' exports/crypto-ecosystems-full-${{ steps.date.outputs.date }}.jsonl | \ - sort -u > exports/ecosystem-list-${{ steps.date.outputs.date }}.txt - echo "✅ Ecosystem list generated" - - - name: Export popular ecosystems individually - run: | - mkdir -p exports/individual - - # List of major ecosystems to export separately - ecosystems=("Bitcoin" "Ethereum" "Solana" "Polkadot" "Cardano" "Avalanche" "Polygon") - - for eco in "${ecosystems[@]}"; do - if [ -d "data/ecosystems/$eco" ] || grep -q "^ecoadd $eco$" migrations/*; then - echo "Exporting $eco..." - ./run.sh export -e "$eco" "exports/individual/${eco,,}-${{ steps.date.outputs.date }}.jsonl" - fi - done - - echo "✅ Individual exports completed" - - - name: Generate statistics JSON - run: | - cat > exports/statistics-${{ steps.date.outputs.date }}.json < exports/README.md < line.trim()) - .map(line => JSON.parse(line)); - \`\`\` - - ## Format - - Each line is a JSON object with the following structure: - \`\`\`json - { - "eco_name": "Bitcoin", - "branch": ["Lightning"], - "repo_url": "https://github.com/lightningnetwork/lnd", - "tags": ["#protocol"] - } - \`\`\` - - ## License - - This data is licensed under MIT License. See LICENSE file for details. - EOF - - echo "✅ Export README generated" - - - name: Create release archive - run: | - cd exports - tar -czf crypto-ecosystems-${{ steps.date.outputs.date }}.tar.gz *.jsonl *.txt *.json *.md individual/ - cd .. - echo "✅ Archive created" - - - name: Upload artifacts - uses: actions/upload-artifact@v4 - with: - name: crypto-ecosystems-export-${{ steps.date.outputs.date }} - path: | - exports/*.jsonl - exports/*.txt - exports/*.json - exports/*.md - exports/individual/ - retention-days: 90 - - - name: Upload release archive - uses: actions/upload-artifact@v4 - with: - name: crypto-ecosystems-archive-${{ steps.date.outputs.date }} - path: exports/*.tar.gz - retention-days: 365 - - - name: Create release (on schedule or manual trigger) - if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' - uses: softprops/action-gh-release@v1 - with: - tag_name: data-export-${{ steps.date.outputs.date }} - name: Data Export ${{ steps.date.outputs.date }} - body: | - Automated weekly export of crypto ecosystem data. - - **Statistics:** - - Total Repositories: $(jq -s 'length' exports/crypto-ecosystems-full-${{ steps.date.outputs.date }}.jsonl) - - Total Ecosystems: $(jq -r '.eco_name' exports/crypto-ecosystems-full-${{ steps.date.outputs.date }}.jsonl | sort -u | wc -l) - - Download the complete archive or individual files below. - files: | - exports/crypto-ecosystems-${{ steps.date.outputs.date }}.tar.gz - exports/crypto-ecosystems-full-${{ steps.date.outputs.date }}.jsonl - exports/statistics-${{ steps.date.outputs.date }}.json - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml deleted file mode 100644 index aa0e787034..0000000000 --- a/.github/workflows/validate.yml +++ /dev/null @@ -1,222 +0,0 @@ -name: Validate Crypto Ecosystems - -on: - push: - branches: [ master, main ] - pull_request: - branches: [ master, main ] - schedule: - # Run daily at 2 AM UTC to check for broken links - - cron: '0 2 * * *' - -jobs: - validate-migrations: - runs-on: ubuntu-latest - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Setup Zig - uses: goto-bus-stop/setup-zig@v2 - with: - version: 0.11.0 - - - name: Build project - run: zig build - - - name: Run tests - run: zig build test - - - name: Validate migration files - run: | - echo "Checking migration file naming convention..." - for file in migrations/*; do - if [[ -f "$file" ]]; then - basename=$(basename "$file") - # Check if filename matches YYYY-MM-DDThhmmss_description format - if [[ ! "$basename" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{6}_.+ ]]; then - echo "❌ Invalid migration filename: $basename" - echo "Expected format: YYYY-MM-DDThhmmss_description" - exit 1 - fi - fi - done - echo "✅ All migration files follow naming convention" - - - name: Export ecosystems - run: | - ./run.sh export test_output.jsonl - if [ ! -f test_output.jsonl ]; then - echo "❌ Export failed - no output file generated" - exit 1 - fi - echo "✅ Export successful" - - - name: Validate JSON output - run: | - echo "Validating JSONL output..." - line_count=0 - while IFS= read -r line; do - line_count=$((line_count + 1)) - if ! echo "$line" | jq empty 2>/dev/null; then - echo "❌ Invalid JSON at line $line_count: $line" - exit 1 - fi - done < test_output.jsonl - echo "✅ All $line_count lines are valid JSON" - - - name: Check for duplicate repositories - run: | - echo "Checking for duplicate repository URLs..." - duplicates=$(jq -r '.repo_url' test_output.jsonl | sort | uniq -d) - if [ ! -z "$duplicates" ]; then - echo "❌ Duplicate repositories found:" - echo "$duplicates" - exit 1 - fi - echo "✅ No duplicate repositories found" - - - name: Generate statistics - run: | - echo "📊 Ecosystem Statistics:" - echo "========================" - total_repos=$(jq -s 'length' test_output.jsonl) - total_ecosystems=$(jq -r '.eco_name' test_output.jsonl | sort -u | wc -l) - echo "Total Ecosystems: $total_ecosystems" - echo "Total Repositories: $total_repos" - echo "" - echo "Top 10 Ecosystems by Repository Count:" - jq -r '.eco_name' test_output.jsonl | sort | uniq -c | sort -rn | head -10 - - - name: Upload export artifact - uses: actions/upload-artifact@v4 - with: - name: ecosystem-export - path: test_output.jsonl - retention-days: 30 - - check-links: - runs-on: ubuntu-latest - if: github.event_name == 'schedule' || github.event_name == 'pull_request' - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Setup Zig - uses: goto-bus-stop/setup-zig@v2 - with: - version: 0.11.0 - - - name: Build and export - run: | - zig build - ./run.sh export repos.jsonl - - - name: Setup Node.js - uses: actions/setup-node@v4 - with: - node-version: '20' - - - name: Check repository accessibility - run: | - echo "Checking repository URLs..." - broken_links=() - checked=0 - - while IFS= read -r line; do - url=$(echo "$line" | jq -r '.repo_url') - checked=$((checked + 1)) - - # Rate limit: check every 2 seconds - if [ $((checked % 10)) -eq 0 ]; then - echo "Checked $checked repositories..." - sleep 2 - fi - - # Check if URL is accessible (HTTP 200 or 301/302 redirect) - status=$(curl -o /dev/null -s -w "%{http_code}" -L "$url" --max-time 10) - - if [ "$status" != "200" ] && [ "$status" != "301" ] && [ "$status" != "302" ]; then - broken_links+=("$url (HTTP $status)") - fi - done < repos.jsonl - - if [ ${#broken_links[@]} -gt 0 ]; then - echo "⚠️ Found ${#broken_links[@]} potentially broken links:" - printf '%s\n' "${broken_links[@]}" - # Don't fail the build, just warn - exit 0 - else - echo "✅ All $checked repository URLs are accessible" - fi - - lint-migrations: - runs-on: ubuntu-latest - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Check migration syntax - run: | - echo "Linting migration files..." - errors=0 - - for file in migrations/*; do - if [[ -f "$file" ]]; then - echo "Checking $file..." - - # Check for valid keywords - while IFS= read -r line; do - # Skip comments and empty lines - if [[ "$line" =~ ^[[:space:]]*-- ]] || [[ -z "$line" ]]; then - continue - fi - - # Check if line starts with valid keyword - if [[ ! "$line" =~ ^(ecoadd|repadd|ecocon)[[:space:]] ]]; then - echo "❌ Invalid syntax in $file: $line" - errors=$((errors + 1)) - fi - - # Validate repadd format (keyword ecosystem url [tag]) - if [[ "$line" =~ ^repadd[[:space:]] ]]; then - if [[ ! "$line" =~ ^repadd[[:space:]]+[^[:space:]]+[[:space:]]+https?:// ]]; then - echo "❌ Invalid repadd format in $file: $line" - errors=$((errors + 1)) - fi - fi - done < "$file" - fi - done - - if [ $errors -gt 0 ]; then - echo "❌ Found $errors syntax errors" - exit 1 - fi - echo "✅ All migration files have valid syntax" - - security-scan: - runs-on: ubuntu-latest - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Run Trivy security scanner - uses: aquasecurity/trivy-action@master - with: - scan-type: 'fs' - scan-ref: '.' - format: 'sarif' - output: 'trivy-results.sarif' - - - name: Upload Trivy results to GitHub Security - uses: github/codeql-action/upload-sarif@v3 - if: always() - with: - sarif_file: 'trivy-results.sarif' \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md deleted file mode 100644 index 70ddec72e6..0000000000 --- a/CONTRIBUTING.md +++ /dev/null @@ -1,476 +0,0 @@ -\# Contributing to Crypto Ecosystems - - - -Thank you for your interest in contributing to Crypto Ecosystems! 🎉 - - - -This guide will help you add new ecosystems, repositories, and make improvements to the project. - - - -\## Table of Contents - -\- \[Getting Started](#getting-started) - -\- \[Adding New Ecosystems](#adding-new-ecosystems) - -\- \[Adding Repositories](#adding-repositories) - -\- \[Connecting Ecosystems](#connecting-ecosystems) - -\- \[Migration File Format](#migration-file-format) - -\- \[Testing Your Changes](#testing-your-changes) - -\- \[Submitting Your Contribution](#submitting-your-contribution) - - - -\## Getting Started - - - -1\. \*\*Fork the repository\*\* to your GitHub account - -2\. \*\*Clone your fork\*\* locally: - -  ```bash - -  git clone https://github.com/YOUR\_USERNAME/crypto-ecosystems.git - -  cd crypto-ecosystems - -  ``` - -3\. \*\*Create a new branch\*\* for your changes: - -  ```bash - -  git checkout -b add-new-ecosystem - -  ``` - - - -\## Adding New Ecosystems - - - -To add a new ecosystem, create a migration file in the `migrations/` directory. - - - -\### Migration File Naming Convention - -``` - -migrations/YYYY-MM-DDThhmmss\_description\_of\_your\_migration - -``` - - - -\*\*Example:\*\* - -``` - -migrations/2025-10-03T153000\_add\_solana\_ecosystem - -``` - - - -\### Migration File Content - - - -Use the Domain Specific Language (DSL) keywords: - - - -\#### 1. \*\*Add Ecosystem\*\* - `ecoadd` - -``` - --- Add a new ecosystem - -ecoadd Solana - -``` - - - -\#### 2. \*\*Add Repository\*\* - `repadd` - -``` - --- Add repositories to the ecosystem - -repadd Solana https://github.com/solana-labs/solana #protocol - -repadd Solana https://github.com/project-serum/serum-dex #defi - -``` - - - -\*\*Available Tags:\*\* - -\- `#protocol` - Core protocol repositories - -\- `#defi` - Decentralized Finance - -\- `#nft` - Non-Fungible Tokens - -\- `#wallet` - Wallet implementations - -\- `#developer-tool` - Development tools - -\- `#bridge` - Cross-chain bridges - -\- `#oracle` - Oracle services - -\- `#dao` - Decentralized Autonomous Organizations - - - -\#### 3. \*\*Connect Ecosystems\*\* - `ecocon` - -``` - --- Connect sub-ecosystems to parent ecosystems - -ecocon Ethereum Uniswap - -ecocon Bitcoin Lightning - -``` - - - -\### Complete Example Migration File - - - -\*\*File:\*\* `migrations/2025-10-03T153000\_add\_solana\_ecosystem` - - - -``` - --- Add Solana ecosystem - -ecoadd Solana - - - --- Add core protocol repos - -repadd Solana https://github.com/solana-labs/solana #protocol - -repadd Solana https://github.com/solana-labs/solana-program-library #protocol - - - --- Add DeFi projects - -repadd Solana https://github.com/project-serum/serum-dex #defi - -repadd Solana https://github.com/solana-labs/solana-web3.js #developer-tool - - - --- Add sub-ecosystem - -ecoadd Serum - -repadd Serum https://github.com/project-serum/serum-ts #developer-tool - -ecocon Solana Serum - -``` - - - -\## Adding Repositories - - - -If you want to add repositories to an \*\*existing ecosystem\*\*, simply create a migration file with `repadd` commands: - - - -\*\*File:\*\* `migrations/2025-10-03T154500\_add\_ethereum\_repos` - - - -``` - --- Add new Ethereum repositories - -repadd Ethereum https://github.com/ethereum/go-ethereum #protocol - -repadd Ethereum https://github.com/OpenZeppelin/openzeppelin-contracts #developer-tool - -repadd Ethereum https://github.com/Uniswap/v3-core #defi - -``` - - - -\## Connecting Ecosystems - - - -To establish parent-child relationships between ecosystems: - - - -``` - --- Polygon is a sub-ecosystem of Ethereum - -ecoadd Polygon - -ecocon Ethereum Polygon - - - --- Optimism is a Layer 2 of Ethereum - -ecoadd Optimism - -ecocon Ethereum Optimism - -``` - - - -\## Migration File Format - - - -\### Rules: - -1\. \*\*Comments\*\* start with `--` - -2\. \*\*One command per line\*\* - -3\. \*\*Datetime format:\*\* `YYYY-MM-DDThhmmss` (ISO 8601 without colons) - -4\. \*\*No special characters\*\* in filenames except underscore `\_` - -5\. \*\*Descriptive names\*\* for clarity - - - -\### Good Examples: - -``` - -migrations/2025-10-03T120000\_add\_bitcoin\_lightning - -migrations/2025-10-03T130000\_add\_defi\_protocols - -migrations/2025-10-03T140000\_connect\_layer2\_networks - -``` - - - -\### Bad Examples: - -``` - -migrations/2025-10-03\_new\_stuff - -migrations/add\_repos - -migrations/2025:10:03T12:00:00\_update - -``` - - - -\## Testing Your Changes - - - -Before submitting, test your migration: - - - -\### On Linux/Mac: - -```bash - -./run.sh export test\_output.jsonl - -``` - - - -\### On Windows: - -```bash - -\# Install Zig if not already installed - -\# Then run: - -zig build run -- export test\_output.jsonl - -``` - - - -Check if your changes appear in the output: - -```bash - -\# On Linux/Mac - -cat test\_output.jsonl | grep "your\_ecosystem\_name" - - - -\# On Windows - -findstr "your\_ecosystem\_name" test\_output.jsonl - -``` - - - -\## Submitting Your Contribution - - - -1\. \*\*Commit your changes:\*\* - -  ```bash - -  git add migrations/ - -  git commit -m "Add Solana ecosystem with core repositories" - -  ``` - - - -2\. \*\*Push to your fork:\*\* - -  ```bash - -  git push origin add-new-ecosystem - -  ``` - - - -3\. \*\*Create a Pull Request:\*\* - -  - Go to the original repository on GitHub - -  - Click "New Pull Request" - -  - Select your branch - -  - Fill in the PR template with: - -  - \*\*Description:\*\* What ecosystems/repos you added - -  - \*\*Motivation:\*\* Why they should be included - -  - \*\*Testing:\*\* Confirm you tested the migration - - - -4\. \*\*PR Title Format:\*\* - -  ``` - -  Add \[Ecosystem Name] ecosystem - -  Add repositories to \[Ecosystem Name] - -  Connect \[Child] to \[Parent] ecosystem - -  ``` - - - -\## Quality Guidelines - - - -\### Repository Requirements: - -\- ✅ Must be \*\*open source\*\* - -\- ✅ Must be related to \*\*blockchain/crypto/web3\*\* - -\- ✅ Must be \*\*actively maintained\*\* (or historically significant) - -\- ✅ Must have a \*\*valid GitHub URL\*\* - -\- ❌ No private repositories - -\- ❌ No spam or low-quality projects - - - -\### Ecosystem Requirements: - -\- ✅ Must be a \*\*recognized blockchain ecosystem\*\* - -\- ✅ Must have multiple \*\*active repositories\*\* - -\- ✅ Must be \*\*publicly documented\*\* - - - -\## Need Help? - - - -\- 📖 Check the \[README.md](README.md) for usage examples - -\- 🐛 Report bugs in \[Issues](https://github.com/PROFADAM/crypto-ecosystems/issues) - -\- 💬 Ask questions in \[Discussions](https://github.com/PROFADAM/crypto-ecosystems/discussions) - -\- 📧 Contact: \[Your Contact Info] - - - -\## Code of Conduct - - - -\- Be respectful and constructive - -\- Focus on the technology, not personal opinions - -\- Provide accurate information - -\- Help others learn and contribute - - - -\## License - - - -By contributing, you agree that your contributions will be licensed under the MIT License. - - - ---- - - - -Thank you for helping build the most comprehensive crypto ecosystem taxonomy! 🚀 - diff --git a/README.md b/README.md index 851060113f..c19f437409 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,8 @@ ecoadd Lightning repadd Lightning https://github.com/lightningnetwork/lnd #protocol -- Connect ecosystems using the ecocon keyword. -- The following connects Lighting as a sub ecosystem of Bitcoin. -ecocon Bitcoin Lightning``` +ecocon Bitcoin Lighting +``` ## How to Give Attribution For Usage of the Electric Capital Crypto Ecosystems diff --git a/api/README.md b/api/README.md deleted file mode 100644 index 7ef1659aca..0000000000 --- a/api/README.md +++ /dev/null @@ -1,622 +0,0 @@ -\# Crypto Ecosystems REST API - - - -A Flask-based REST API for accessing crypto ecosystem taxonomy data. - - - -\## Installation - - - -\### Prerequisites - -\- Python 3.8 or higher - -\- Zig (for building the main project) - - - -\### Setup - - - -1\. \*\*Install Python dependencies:\*\* - -  ```bash - -  cd api - -  pip install -r requirements.txt - -  ``` - - - -2\. \*\*Build the main project (required for data export):\*\* - -  ```bash - -  cd .. - -  zig build - -  ``` - - - -\## Running the API - - - -\### Development Mode - -```bash - -cd api - -python app.py - -``` - - - -The API will start at `http://localhost:5000` - - - -\### Production Mode (with Gunicorn) - -```bash - -pip install gunicorn - -gunicorn -w 4 -b 0.0.0.0:5000 app:app - -``` - - - -\## API Endpoints - - - -\### 📚 Documentation - -``` - -GET / - -``` - -Returns API documentation and available endpoints. - - - -\### 🌐 Get All Ecosystems - -``` - -GET /api/ecosystems - -``` - -Returns a list of all ecosystems with repository counts. - - - -\*\*Response:\*\* - -```json - -{ - -  "total": 150, - -  "ecosystems": \[ - -  { - -  "name": "Bitcoin", - -  "repository\_count": 523, - -  "sub\_ecosystem\_count": 5, - -  "sub\_ecosystems": \["Lightning", "Liquid"] - -  } - -  ] - -} - -``` - - - -\### 🔍 Get Specific Ecosystem - -``` - -GET /api/ecosystems/ - -``` - -Returns detailed information about a specific ecosystem. - - - -\*\*Example:\*\* - -```bash - -curl http://localhost:5000/api/ecosystems/Bitcoin - -``` - - - -\*\*Response:\*\* - -```json - -{ - -  "name": "Bitcoin", - -  "repository\_count": 523, - -  "repositories": \["https://github.com/bitcoin/bitcoin", ...], - -  "sub\_ecosystems": \["Lightning", "Liquid"], - -  "tags": { - -  "#protocol": 45, - -  "#wallet": 32, - -  "#developer-tool": 28 - -  }, - -  "sample\_repos": \[...] - -} - -``` - - - -\### 📦 Get All Repositories - -``` - -GET /api/repositories?page=1\&per\_page=50 - -``` - -Returns paginated list of all repositories. - - - -\*\*Query Parameters:\*\* - -\- `page` (default: 1) - Page number - -\- `per\_page` (default: 50, max: 200) - Items per page - - - -\*\*Response:\*\* - -```json - -{ - -  "total": 15234, - -  "page": 1, - -  "per\_page": 50, - -  "total\_pages": 305, - -  "repositories": \[...] - -} - -``` - - - -\### 🔎 Search Repositories - -``` - -GET /api/repositories/search?q= - -``` - -Search repositories by name, ecosystem, or tags. - - - -\*\*Example:\*\* - -```bash - -curl http://localhost:5000/api/repositories/search?q=ethereum - -``` - - - -\*\*Response:\*\* - -```json - -{ - -  "query": "ethereum", - -  "total": 234, - -  "results": \[...] - -} - -``` - - - -\### 🏷️ Get All Tags - -``` - -GET /api/tags - -``` - -Returns all available tags with their usage counts. - - - -\*\*Response:\*\* - -```json - -{ - -  "total\_tags": 15, - -  "tags": \[ - -  {"tag": "#protocol", "count": 423}, - -  {"tag": "#defi", "count": 312}, - -  {"tag": "#wallet", "count": 245} - -  ] - -} - -``` - - - -\### 📊 Get Statistics - -``` - -GET /api/stats - -``` - -Returns overall statistics about the dataset. - - - -\*\*Response:\*\* - -```json - -{ - -  "timestamp": "2025-10-03T16:30:00", - -  "total\_ecosystems": 156, - -  "total\_repositories": 15234, - -  "total\_tags": 15, - -  "top\_ecosystems": \[ - -  {"name": "Ethereum", "repository\_count": 3245}, - -  {"name": "Bitcoin", "repository\_count": 1523} - -  ], - -  "tag\_distribution": \[...], - -  "data\_freshness": { - -  "cached": true, - -  "cache\_age\_seconds": 245 - -  } - -} - -``` - - - -\### ❤️ Health Check - -``` - -GET /api/health - -``` - -Returns API health status. - - - -\*\*Response:\*\* - -```json - -{ - -  "status": "healthy", - -  "timestamp": "2025-10-03T16:30:00", - -  "cache\_age\_seconds": 245 - -} - -``` - - - -\## Usage Examples - - - -\### Python - -```python - -import requests - - - -\# Get all ecosystems - -response = requests.get('http://localhost:5000/api/ecosystems') - -ecosystems = response.json() - - - -\# Search for Ethereum repositories - -response = requests.get('http://localhost:5000/api/repositories/search?q=ethereum') - -results = response.json() - - - -\# Get statistics - -response = requests.get('http://localhost:5000/api/stats') - -stats = response.json() - -print(f"Total repos: {stats\['total\_repositories']}") - -``` - - - -\### JavaScript (Node.js) - -```javascript - -const axios = require('axios'); - - - -async function getEcosystems() { - -  const response = await axios.get('http://localhost:5000/api/ecosystems'); - -  console.log(response.data); - -} - - - -async function searchRepos(query) { - -  const response = await axios.get( - -  `http://localhost:5000/api/repositories/search?q=${query}` - -  ); - -  return response.data.results; - -} - -``` - - - -\### cURL - -```bash - -\# Get all ecosystems - -curl http://localhost:5000/api/ecosystems - - - -\# Get Bitcoin ecosystem details - -curl http://localhost:5000/api/ecosystems/Bitcoin - - - -\# Search for DeFi projects - -curl "http://localhost:5000/api/repositories/search?q=defi" - - - -\# Get statistics - -curl http://localhost:5000/api/stats - -``` - - - -\## Caching - - - -The API caches exported data for 1 hour (3600 seconds) to improve performance. After the cache expires, data is automatically refreshed from the latest migrations. - - - -\## CORS - - - -CORS is enabled for all origins, making the API accessible from web applications. - - - -\## Error Handling - - - -The API returns standard HTTP status codes: - -\- `200` - Success - -\- `400` - Bad Request (missing or invalid parameters) - -\- `404` - Not Found (ecosystem or endpoint doesn't exist) - -\- `500` - Internal Server Error - - - -Error responses follow this format: - -```json - -{ - -  "error": "Error description" - -} - -``` - - - -\## Docker Deployment - - - -Create a `Dockerfile`: - -```dockerfile - -FROM python:3.11-slim - - - -WORKDIR /app - - - -\# Install Zig - -RUN apt-get update \&\& apt-get install -y wget xz-utils - -RUN wget https://ziglang.org/download/0.11.0/zig-linux-x86\_64-0.11.0.tar.xz - -RUN tar -xf zig-linux-x86\_64-0.11.0.tar.xz - -ENV PATH="/app/zig-linux-x86\_64-0.11.0:${PATH}" - - - -\# Copy project files - -COPY . . - - - -\# Build project - -RUN zig build - - - -\# Install Python dependencies - -RUN pip install -r api/requirements.txt - - - -EXPOSE 5000 - - - -CMD \["python", "api/app.py"] - -``` - - - -Build and run: - -```bash - -docker build -t crypto-ecosystems-api . - -docker run -p 5000:5000 crypto-ecosystems-api - -``` - - - -\## Contributing - - - -See \[CONTRIBUTING.md](../CONTRIBUTING.md) for guidelines on contributing to this project. - - - -\## License - - - -MIT License - See \[LICENSE](../LICENSE) for details. - diff --git a/api/app.py b/api/app.py deleted file mode 100644 index f829344a10..0000000000 --- a/api/app.py +++ /dev/null @@ -1,272 +0,0 @@ -from flask import Flask, jsonify, request -from flask_cors import CORS -import json -import os -import subprocess -from collections import defaultdict, Counter -from datetime import datetime - -app = Flask(__name__) -CORS(app) # Enable CORS for all routes - -# Cache for exported data -_cache = { - 'data': None, - 'timestamp': None, - 'ttl': 3600 # Cache for 1 hour -} - -def load_ecosystem_data(): - """Load ecosystem data from export or cache""" - current_time = datetime.now() - - # Return cached data if still valid - if _cache['data'] and _cache['timestamp']: - time_diff = (current_time - _cache['timestamp']).seconds - if time_diff < _cache['ttl']: - return _cache['data'] - - # Generate fresh export - export_file = 'api_export.jsonl' - - try: - # Run export command - if os.name == 'nt': # Windows - subprocess.run(['cmd', '/c', 'run.sh', 'export', export_file], check=True) - else: # Linux/Mac - subprocess.run(['./run.sh', 'export', export_file], check=True) - - # Load data - data = [] - with open(export_file, 'r', encoding='utf-8') as f: - for line in f: - if line.strip(): - data.append(json.loads(line)) - - # Update cache - _cache['data'] = data - _cache['timestamp'] = current_time - - # Clean up export file - os.remove(export_file) - - return data - except Exception as e: - print(f"Error loading data: {e}") - return _cache['data'] if _cache['data'] else [] - -@app.route('/') -def index(): - """API documentation""" - return jsonify({ - 'name': 'Crypto Ecosystems API', - 'version': '1.0.0', - 'description': 'REST API for crypto ecosystem taxonomy data', - 'endpoints': { - '/': 'API documentation (this page)', - '/api/ecosystems': 'List all ecosystems', - '/api/ecosystems/': 'Get specific ecosystem details', - '/api/repositories': 'List all repositories', - '/api/repositories/search?q=': 'Search repositories', - '/api/stats': 'Get overall statistics', - '/api/tags': 'List all available tags', - '/api/health': 'API health check' - }, - 'documentation': 'https://github.com/PROFADAM/crypto-ecosystems' - }) - -@app.route('/api/health') -def health(): - """Health check endpoint""" - return jsonify({ - 'status': 'healthy', - 'timestamp': datetime.now().isoformat(), - 'cache_age_seconds': (datetime.now() - _cache['timestamp']).seconds if _cache['timestamp'] else None - }) - -@app.route('/api/ecosystems') -def get_ecosystems(): - """Get list of all ecosystems with their repository counts""" - data = load_ecosystem_data() - - # Group by ecosystem - ecosystems = defaultdict(lambda: {'repos': [], 'sub_ecosystems': set()}) - - for item in data: - eco_name = item['eco_name'] - ecosystems[eco_name]['repos'].append(item['repo_url']) - - # Track sub-ecosystems - if item.get('branch'): - for branch in item['branch']: - ecosystems[eco_name]['sub_ecosystems'].add(branch) - - # Format response - result = [] - for name, info in sorted(ecosystems.items()): - result.append({ - 'name': name, - 'repository_count': len(info['repos']), - 'sub_ecosystem_count': len(info['sub_ecosystems']), - 'sub_ecosystems': sorted(list(info['sub_ecosystems'])) - }) - - return jsonify({ - 'total': len(result), - 'ecosystems': result - }) - -@app.route('/api/ecosystems/') -def get_ecosystem(ecosystem_name): - """Get detailed information about a specific ecosystem""" - data = load_ecosystem_data() - - # Filter by ecosystem name (case-insensitive) - ecosystem_data = [ - item for item in data - if item['eco_name'].lower() == ecosystem_name.lower() - ] - - if not ecosystem_data: - return jsonify({'error': 'Ecosystem not found'}), 404 - - # Collect statistics - repos = [item['repo_url'] for item in ecosystem_data] - tags = [] - for item in ecosystem_data: - tags.extend(item.get('tags', [])) - - sub_ecosystems = set() - for item in ecosystem_data: - if item.get('branch'): - sub_ecosystems.update(item['branch']) - - return jsonify({ - 'name': ecosystem_data[0]['eco_name'], - 'repository_count': len(repos), - 'repositories': repos, - 'sub_ecosystems': sorted(list(sub_ecosystems)), - 'tags': dict(Counter(tags)), - 'sample_repos': repos[:10] # First 10 repos as sample - }) - -@app.route('/api/repositories') -def get_repositories(): - """Get all repositories with pagination""" - data = load_ecosystem_data() - - # Pagination - page = request.args.get('page', 1, type=int) - per_page = request.args.get('per_page', 50, type=int) - per_page = min(per_page, 200) # Max 200 per page - - start = (page - 1) * per_page - end = start + per_page - - total = len(data) - repos = data[start:end] - - return jsonify({ - 'total': total, - 'page': page, - 'per_page': per_page, - 'total_pages': (total + per_page - 1) // per_page, - 'repositories': repos - }) - -@app.route('/api/repositories/search') -def search_repositories(): - """Search repositories by query""" - query = request.args.get('q', '').lower() - - if not query: - return jsonify({'error': 'Query parameter "q" is required'}), 400 - - data = load_ecosystem_data() - - # Search in repo URL and ecosystem name - results = [ - item for item in data - if query in item['repo_url'].lower() or - query in item['eco_name'].lower() or - any(query in tag.lower() for tag in item.get('tags', [])) - ] - - return jsonify({ - 'query': query, - 'total': len(results), - 'results': results - }) - -@app.route('/api/tags') -def get_tags(): - """Get all available tags with their counts""" - data = load_ecosystem_data() - - tags = [] - for item in data: - tags.extend(item.get('tags', [])) - - tag_counts = Counter(tags) - - return jsonify({ - 'total_tags': len(tag_counts), - 'tags': [ - {'tag': tag, 'count': count} - for tag, count in tag_counts.most_common() - ] - }) - -@app.route('/api/stats') -def get_stats(): - """Get overall statistics""" - data = load_ecosystem_data() - - # Calculate statistics - ecosystems = set(item['eco_name'] for item in data) - repos = set(item['repo_url'] for item in data) - - tags = [] - for item in data: - tags.extend(item.get('tags', [])) - - # Top ecosystems by repo count - eco_counts = Counter(item['eco_name'] for item in data) - top_ecosystems = [ - {'name': name, 'repository_count': count} - for name, count in eco_counts.most_common(10) - ] - - # Tag distribution - tag_counts = Counter(tags) - tag_distribution = [ - {'tag': tag, 'count': count} - for tag, count in tag_counts.most_common(10) - ] - - return jsonify({ - 'timestamp': datetime.now().isoformat(), - 'total_ecosystems': len(ecosystems), - 'total_repositories': len(repos), - 'total_tags': len(set(tags)), - 'top_ecosystems': top_ecosystems, - 'tag_distribution': tag_distribution, - 'data_freshness': { - 'cached': _cache['timestamp'] is not None, - 'cache_age_seconds': (datetime.now() - _cache['timestamp']).seconds if _cache['timestamp'] else None - } - }) - -@app.errorhandler(404) -def not_found(error): - return jsonify({'error': 'Endpoint not found'}), 404 - -@app.errorhandler(500) -def internal_error(error): - return jsonify({'error': 'Internal server error'}), 500 - -if __name__ == '__main__': - print("🚀 Starting Crypto Ecosystems API...") - print("📡 API will be available at: http://localhost:5000") - print("📚 Documentation: http://localhost:5000/") - app.run(debug=True, host='0.0.0.0', port=5000) \ No newline at end of file diff --git a/api/requirements.txt b/api/requirements.txt deleted file mode 100644 index 60763e5bbc..0000000000 --- a/api/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -Flask==3.0.0 -Flask-CORS==4.0.0 \ No newline at end of file