chapter + tools

Author: max-ostapenko

src/tools/scripts/chart-adjustments.ipynb sql/util/chart-adjustments.ipynbsrc/tools/scripts/chart-adjustments.ipynb renamed to sql/util/chart-adjustments.ipynb

@@ -0,0 +1,78 @@
+import re
+import os
+import sys
+from googleapiclient.discovery import build
+import google.auth
+
+# Configuration
+SPREADSHEET_ID = '1Svyw40Th7VbigX6lpR1lb1WXwTUVKZWrK7O2YELrml4'
+PUBCHART_ID = '2PACX-1vRC5wrzy5NEsWNHn9w38RLsMURRScnP4jgjO1mDiVhsfFCY55tujlTUZhUaEWzmPtJza0QA7w8S4uK5'
+SQL_DIR = '../2025/privacy' # Relative to this script's location
+
+SCOPES = ['https://www.googleapis.com/auth/spreadsheets']
+
+def get_sql_to_sheet_map(sql_dir):
+    mapping = {}
+    if not os.path.exists(sql_dir):
+        print(f"Directory not found: {sql_dir}")
+        return mapping
+    for filename in os.listdir(sql_dir):
+        if filename.endswith(".sql"):
+            # Generate sheet name from filename using the regex:
+            # re.sub(r'(\.sql|[^a-zA-Z0-9]+)', ' ', filename).strip().title()
+            sheet_name = re.sub(r'(\.sql|[^a-zA-Z0-9]+)', ' ', filename).strip().title()
+            mapping[sheet_name] = filename
+    return mapping
+
+def generate_figure_markup(spreadsheet_id, sql_dir):
+    try:
+        credentials, project = google.auth.default(scopes=SCOPES)
+        sheets_service = build('sheets', 'v4', cache_discovery=False, credentials=credentials)
+    except Exception as e:
+        print(f"Authentication failed: {e}")
+        print("Please ensure you have application default credentials set up.")
+        return
+
+    sql_map = get_sql_to_sheet_map(sql_dir)
+    response = sheets_service.spreadsheets().get(spreadsheetId=spreadsheet_id, includeGridData=False).execute()
+    sheets = response.get('sheets', [])
+
+    for sheet in sheets:
+        sheet_name = sheet['properties']['title']
+        sheet_id = sheet['properties']['sheetId']
+        charts = sheet.get('charts', [])
+        
+        sql_file = sql_map.get(sheet_name)
+        if not sql_file:
+            # Try to match case-insensitively or show warning
+            sql_file = "TODO.sql"
+
+        for chart in charts:
+            title = chart['spec'].get('title', 'Untitled Chart')
+            chart_id = chart['chartId']
+            
+            # Slugify for image name
+            image_name = re.sub(r'[^a-z0-9]+', '-', title.lower()).strip('-') + ".png"
+            
+            # Construct markup
+            markup = f"""{{{{ figure_markup(
+  image="{image_name}",
+  caption="{title}",
+  description="",
+  chart_url="https://docs.google.com/spreadsheets/d/e/{PUBCHART_ID}/pubchart?oid={chart_id}&format=interactive",
+  sheets_gid="{sheet_id}",
+  sql_file="{sql_file}"
+  )
+}}}}"""
+            print(markup)
+            print()
+
+if __name__ == "__main__":
+    # Resolve relative SQL_DIR based on script location
+    script_dir = os.path.dirname(os.path.abspath(__file__))
+    absolute_sql_dir = os.path.normpath(os.path.join(script_dir, SQL_DIR))
+    
+    print(f"Processing Spreadsheet: {SPREADSHEET_ID}")
+    print(f"SQL Directory: {absolute_sql_dir}\n")
+    
+    generate_figure_markup(SPREADSHEET_ID, absolute_sql_dir)

sql/util/generate_chart_markup.py

@@ -1,5 +1,6 @@
 pandas==2.3.3
 google-cloud-bigquery==3.40.0
+google-api-python-client==2.188.0
 requests==2.32.5
 tabulate==0.9.0
 gspread==6.2.1

sql/util/requirements.txt

@@ -68,8 +68,7 @@
           "chapter_number": "8",
           "title": "Privacy",
           "slug": "privacy",
-          "hero_dir": "2020",
-          "todo": true
+          "hero_dir": "2020"
         },
         {
           "part": "II",

src/config/2025.json

@@ -3240,6 +3240,10 @@
         "analysts",
         "developers",
         "authors"
+      ],
+      "2025": [
+        "analysts",
+        "editors"
       ]
     },
     "website": "https://maxostapenko.com"
@@ -3769,7 +3773,8 @@
       ],
       "2025": [
         "leads",
-        "committee"
+        "committee",
+        "authors"
       ]
     },
     "twitter": "nrllah",
@@ -5151,6 +5156,7 @@
     "name": "Vinod Tiwari",
     "teams": {
       "2025": [
+        "authors",
         "reviewers"
       ]
     },
@@ -5418,5 +5424,18 @@
       ]
     },
     "twitter": "_cybai"
+  },
+  "RumaisaHabib": {
+    "avatar_url": "66083065",
+    "github": "RumaisaHabib",
+    "name": "Rumaisa Habib",
+    "teams": {
+      "2025": [
+        "authors"
+      ]
+    },
+    "website": "https://rumaisahabib.com/",
+    "linkedin": "rumaisahabib"
   }
+
 }

src/config/contributors.json

@@ -17,6 +17,7 @@
     "test": "node ./tools/test",
     "generate": "node ./tools/generate",
     "figure-images": "node ./tools/generate/generate_figure_images",
+    "chart-descriptions": "node ./tools/generate/generate_chart_descriptions",
     "watch": "node ./tools/generate/chapter_watcher",
     "ebooks": "node ./tools/generate/generate_ebook_pdfs",
     "deploy": "./tools/scripts/deploy.sh",
@@ -47,7 +48,8 @@
     "run-script-os": "1.1.6",
     "showdown": "2.1.0",
     "smartypants": "0.2.2",
+    "tesseract.js": "^7.0.0",
     "web-vitals": "5.1.0",
     "xml-js": "1.6.11"
   }
-}
+}