IOC-Scraper.yml

name: Scrape Domains and Generate CSVs

on:
  schedule:
    - cron: '0 9 * * 1'  # Runs every Monday at 09:00 UTC
  workflow_dispatch:  # Allows manual trigger

jobs:
  scrape:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: '3.x'

      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install requests beautifulsoup4 pandas

      - name: Ensure error log exists
        run: |
          mkdir -p Defender/IOCs/error-log
          touch Defender/IOCs/error-log/.gitkeep

      - name: Run scraper script
        run: python .github/workflows/scripts/scrape_domains.py

      - name: Commit and push changes
        run: |
          # Set Git configuration
          git config --global user.name "github-actions[bot]"
          git config --global user.email "github-actions[bot]@users.noreply.github.com"

          # Fetch the latest changes from the remote
          git fetch origin

          # Checkout a new branch for updates
          git checkout -b update-domain-lists

          # Stage changes
          git add Defender/IOCs/*.csv Defender/IOCs/error-log/*
          git commit -m "Update domain lists and error logs"

          # Attempt to rebase with the latest changes if the branch exists
          if git show-ref --verify --quiet refs/remotes/origin/main; then
            if ! git rebase origin/main; then
              echo "Rebase conflicts occurred. Please resolve them manually."
              exit 1
            fi
          else
            echo "No existing remote branch 'main'. Skipping rebase."
          fi

          # Push the new branch to the remote
          if ! git push --set-upstream https://${{ secrets.PAT_TOKEN }}@github.com/Marshyp/Security-Scripts.git update-domain-lists; then
            echo "Push failed, trying to fetch and rebase again."
            git fetch origin
            if git show-ref --verify --quiet refs/remotes/origin/main; then
              if ! git rebase origin/main; then
                echo "Second rebase failed. Please resolve manually."
                exit 1
              fi
            else
              echo "No existing remote branch 'main'. Skipping rebase."
            fi
            if ! git push --set-upstream https://${{ secrets.PAT_TOKEN }}@github.com/Marshyp/Security-Scripts.git update-domain-lists; then
              echo "Second push failed. Please resolve manually."
              exit 1
            fi
          fi
        continue-on-error: true

      - name: Create Pull Request
        run: |
          curl -H "Authorization: token ${{ secrets.PAT_TOKEN }}" \
              -X POST \
              -d '{"title": "Update domain lists", "head": "update-domain-lists", "base": "main"}' \
              https://api.github.com/repos/${{ github.repository }}/pulls