crawler.yml

name: Get Hot News

on:
  schedule:
    # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
    # ⚠️ 试用版说明 / Trial Mode
    # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
    #
    # 🔄 运行机制 / How it works:
    #    - 每个周期为 7 天，届时自动停止
    #    - 运行 "Check In" 会重置周期（重新开始 7 天倒计时，而非累加）
    #    - Each cycle is 7 days, then auto-stops
    #    - "Check In" resets the cycle (restarts 7-day countdown, not cumulative)
    #
    # 💡 设计初衷 / Why this design:
    #    如果 7 天都忘了签到，或许这些资讯对你来说并非刚需
    #    适时的暂停，能帮你从信息流中抽离，给大脑留出喘息的空间
    #    If you forget for 7 days, maybe you don't really need it
    #    A timely pause helps you detach from the stream and gives your mind space
    #
    # 🙏 珍惜资源 / Respect shared resources:
    #    GitHub Actions 是平台提供的公共资源，每次运行都会消耗算力
    #    签到机制确保资源分配给真正需要的用户，感谢你的理解与配合
    #    GitHub Actions is a shared public resource provided by the platform
    #    Check-in ensures resources go to those who truly need it — thank you
    #
    # 🚀 长期使用请部署 Docker 版本 / For long-term use, deploy Docker version
    #
    # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
    #
    # 📝 修改运行时间：只改第一个数字（0-59），表示每小时第几分钟运行
    # 📝 Change time: Only modify the first number (0-59) = minute of each hour
    #
    # 示例 / Examples:
    #   "15 * * * *"     → 每小时第15分钟 / minute 15 every hour
    #   "30 0-14 * * *"  → 北京时间 8:00-22:00 每小时第30分钟 / Beijing 8am-10pm
    #
    - cron: "33 * * * *"

  workflow_dispatch:

concurrency:
  group: crawler-${{ github.ref_name }}
  cancel-in-progress: true

permissions:
  contents: read
  actions: write

jobs:
  crawl:
    runs-on: ubuntu-latest
    timeout-minutes: 15

    steps:
      - name: Checkout repository
        uses: actions/checkout@v6
        with:
          fetch-depth: 1
          clean: true

      - name: Check Expiration
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          WORKFLOW_FILE="crawler.yml"
          API_URL="repos/${{ github.repository }}/actions/workflows/$WORKFLOW_FILE/runs"

          TOTAL=$(gh api "$API_URL" --jq '.total_count')
          if [ -z "$TOTAL" ] || [ "$TOTAL" -eq 0 ]; then
            echo "No previous runs found, skipping expiration check"
            exit 0
          fi

          LAST_PAGE=$(( (TOTAL + 99) / 100 ))
          FIRST_RUN_DATE=$(gh api "$API_URL?per_page=100&page=$LAST_PAGE" --jq '.workflow_runs[-1].created_at')

          if [ -n "$FIRST_RUN_DATE" ]; then
            CURRENT_TIMESTAMP=$(date +%s)
            FIRST_RUN_TIMESTAMP=$(date -d "$FIRST_RUN_DATE" +%s)
            DIFF_SECONDS=$((CURRENT_TIMESTAMP - FIRST_RUN_TIMESTAMP))
            LIMIT_SECONDS=604800

            if [ $DIFF_SECONDS -gt $LIMIT_SECONDS ]; then
              echo "⚠️ 试用期已结束，请运行 'Check In' 签到续期"
              echo "⚠️ Trial expired. Run 'Check In' to renew."
              gh workflow disable "$WORKFLOW_FILE"
              exit 1
            else
              DAYS_LEFT=$(( (LIMIT_SECONDS - DIFF_SECONDS) / 86400 ))
              echo "✅ 试用期剩余 ${DAYS_LEFT} 天，到期前请运行 'Check In' 签到续期"
              echo "✅ Trial: ${DAYS_LEFT} days left. Run 'Check In' before expiry to renew."
            fi
          fi


      # --------------------------------------------------------------------------------
      # 🚦 TRAFFIC CONTROL / 流量控制
      # --------------------------------------------------------------------------------
      # EN: Generates a random delay between 1 and 300 seconds (5 minutes).
      #     Critical for load balancing.
      #
      # CN: 生成 1 到 300 秒（5分钟）之间的随机延迟。
      #     这对负载均衡至关重要。
      # - name: Random Delay (Traffic Control)
      #   if: success()
      #   run: |
      #     echo "🎲 Traffic Control: Generating random delay..."
      #     DELAY=$(( ( RANDOM % 300 )  + 1 ))
      #     echo "⏸️  Sleeping for ${DELAY} seconds to spread the load..."
      #     sleep ${DELAY}s
      #     echo "▶️  Delay finished. Starting crawler..."

      - name: Set up Python
        if: success()
        uses: actions/setup-python@v6
        with:
          python-version: "3.12"

      - name: Install uv
        if: success()
        uses: astral-sh/setup-uv@v7

      - name: Install dependencies
        if: success()
        run: uv sync --frozen --no-dev

      - name: Verify required files
        if: success()
        run: |
          if [ ! -f config/config.yaml ]; then
            echo "Error: Config missing"
            exit 1
          fi

      - name: Run crawler
        if: success()
        env:
          FEISHU_WEBHOOK_URL: ${{ secrets.FEISHU_WEBHOOK_URL }}
          TELEGRAM_BOT_TOKEN: ${{ secrets.TELEGRAM_BOT_TOKEN }}
          TELEGRAM_CHAT_ID: ${{ secrets.TELEGRAM_CHAT_ID }}
          DINGTALK_WEBHOOK_URL: ${{ secrets.DINGTALK_WEBHOOK_URL }}
          WEWORK_WEBHOOK_URL: ${{ secrets.WEWORK_WEBHOOK_URL }}
          WEWORK_MSG_TYPE: ${{ secrets.WEWORK_MSG_TYPE }}
          EMAIL_FROM: ${{ secrets.EMAIL_FROM }}
          EMAIL_PASSWORD: ${{ secrets.EMAIL_PASSWORD }}
          EMAIL_TO: ${{ secrets.EMAIL_TO }}
          EMAIL_SMTP_SERVER: ${{ secrets.EMAIL_SMTP_SERVER }}
          EMAIL_SMTP_PORT: ${{ secrets.EMAIL_SMTP_PORT }}
          NTFY_TOPIC: ${{ secrets.NTFY_TOPIC }}
          NTFY_SERVER_URL: ${{ secrets.NTFY_SERVER_URL }}
          NTFY_TOKEN: ${{ secrets.NTFY_TOKEN }}
          BARK_URL: ${{ secrets.BARK_URL }}
          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
          # 通用Webhook配置
          GENERIC_WEBHOOK_URL: ${{ secrets.GENERIC_WEBHOOK_URL }}
          GENERIC_WEBHOOK_TEMPLATE: ${{ secrets.GENERIC_WEBHOOK_TEMPLATE }}
          # AI 配置（ai_analysis 和 ai_translation 共享模型配置）
          AI_ANALYSIS_ENABLED: ${{ secrets.AI_ANALYSIS_ENABLED }}
          AI_API_KEY: ${{ secrets.AI_API_KEY }}
          AI_MODEL: ${{ secrets.AI_MODEL }}
          AI_API_BASE: ${{ secrets.AI_API_BASE }}
          # 远程存储配置
          S3_BUCKET_NAME: ${{ secrets.S3_BUCKET_NAME }}
          S3_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }}
          S3_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_ACCESS_KEY }}
          S3_ENDPOINT_URL: ${{ secrets.S3_ENDPOINT_URL }}
          S3_REGION: ${{ secrets.S3_REGION }}
          GITHUB_ACTIONS: true
        run: uv run python -m trendradar

      # --------------------------------------------------------------------------------
      # ☁️ CLOUDFLARE PAGES DEPLOY / 部署到 Cloudflare Pages（可选）
      # --------------------------------------------------------------------------------
      # 将爬虫生成的 index.html 自动部署到 Cloudflare Pages，国内访问比 GitHub Pages 更快。
      # 仅当下面三个 Secrets 均已配置时才会执行，缺任意一个则自动跳过，不影响其它流程：
      #   - CLOUDFLARE_API_TOKEN     权限：Account → Cloudflare Pages → Edit
      #   - CLOUDFLARE_ACCOUNT_ID    Cloudflare 账户 ID
      #   - CLOUDFLARE_PROJECT_NAME  Cloudflare Pages 项目名（需提前在 Dashboard 创建）
      # 配置教程详见 README「自动部署到 Cloudflare Pages」章节。
      # --------------------------------------------------------------------------------
      - name: Check Cloudflare config
        if: success()
        id: cf_check
        env:
          CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
          CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
          CLOUDFLARE_PROJECT_NAME: ${{ secrets.CLOUDFLARE_PROJECT_NAME }}
        run: |
          if [ -n "$CLOUDFLARE_API_TOKEN" ] && [ -n "$CLOUDFLARE_ACCOUNT_ID" ] && [ -n "$CLOUDFLARE_PROJECT_NAME" ]; then
            echo "enabled=true" >> "$GITHUB_OUTPUT"
            echo "✅ 检测到 Cloudflare 配置，准备部署到 Cloudflare Pages"
          else
            echo "enabled=false" >> "$GITHUB_OUTPUT"
            echo "ℹ️ 未配置 Cloudflare Secrets，跳过 Cloudflare Pages 部署"
          fi

      - name: Prepare Cloudflare Pages assets
        if: success() && steps.cf_check.outputs.enabled == 'true'
        run: |
          if [ ! -f index.html ]; then
            echo "❌ 未找到 index.html，请确认爬虫已生成网页报告"
            exit 1
          fi
          rm -rf cloudflare_dist
          mkdir -p cloudflare_dist
          cp index.html cloudflare_dist/index.html

      - name: Deploy to Cloudflare Pages
        if: success() && steps.cf_check.outputs.enabled == 'true'
        uses: cloudflare/wrangler-action@v3
        with:
          apiToken: ${{ secrets.CLOUDFLARE_API_TOKEN }}
          accountId: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
          command: pages deploy cloudflare_dist --project-name=${{ secrets.CLOUDFLARE_PROJECT_NAME }}