Scrape Domains and Generate CSVs #37
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Scrape Domains and Generate CSVs | |
| on: | |
| schedule: | |
| - cron: '0 9 * * 1' # Runs every Monday at 09:00 UTC | |
| workflow_dispatch: # Allows manual trigger | |
| jobs: | |
| scrape: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.x' | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install requests beautifulsoup4 pandas | |
| - name: Ensure error log exists | |
| run: | | |
| mkdir -p Defender/IOCs/error-log | |
| touch Defender/IOCs/error-log/.gitkeep | |
| - name: Run scraper script | |
| run: python .github/workflows/scripts/scrape_domains.py | |
| - name: Commit and push changes | |
| run: | | |
| # Set Git configuration | |
| git config --global user.name "github-actions[bot]" | |
| git config --global user.email "github-actions[bot]@users.noreply.github.com" | |
| # Fetch the latest changes from the remote | |
| git fetch origin | |
| # Checkout a new branch for updates | |
| git checkout -b update-domain-lists | |
| # Stage changes | |
| git add Defender/IOCs/*.csv Defender/IOCs/error-log/* | |
| git commit -m "Update domain lists and error logs" | |
| # Attempt to rebase with the latest changes if the branch exists | |
| if git show-ref --verify --quiet refs/remotes/origin/main; then | |
| if ! git rebase origin/main; then | |
| echo "Rebase conflicts occurred. Please resolve them manually." | |
| exit 1 | |
| fi | |
| else | |
| echo "No existing remote branch 'main'. Skipping rebase." | |
| fi | |
| # Push the new branch to the remote | |
| if ! git push --set-upstream https://${{ secrets.PAT_TOKEN }}@github.com/Marshyp/Security-Scripts.git update-domain-lists; then | |
| echo "Push failed, trying to fetch and rebase again." | |
| git fetch origin | |
| if git show-ref --verify --quiet refs/remotes/origin/main; then | |
| if ! git rebase origin/main; then | |
| echo "Second rebase failed. Please resolve manually." | |
| exit 1 | |
| fi | |
| else | |
| echo "No existing remote branch 'main'. Skipping rebase." | |
| fi | |
| if ! git push --set-upstream https://${{ secrets.PAT_TOKEN }}@github.com/Marshyp/Security-Scripts.git update-domain-lists; then | |
| echo "Second push failed. Please resolve manually." | |
| exit 1 | |
| fi | |
| fi | |
| continue-on-error: true | |
| - name: Create Pull Request | |
| run: | | |
| curl -H "Authorization: token ${{ secrets.PAT_TOKEN }}" \ | |
| -X POST \ | |
| -d '{"title": "Update domain lists", "head": "update-domain-lists", "base": "main"}' \ | |
| https://api.github.com/repos/${{ github.repository }}/pulls | |