First run of code

src/gui/components/log_viewer.py

@@ -0,0 +1,23 @@
+# src/gui/components/log_viewer.py
+
+from PySide6.QtWidgets import QWidget, QVBoxLayout, QLabel, QTextEdit
+
+class LogViewer(QWidget):
+    def __init__(self, title="Live Extraction Logs", parent=None):
+        super().__init__(parent)
+        layout = QVBoxLayout(self)
+        layout.setContentsMargins(0, 0, 0, 0)
+
+        self.lbl_title = QLabel(title)
+        self.txt_log = QTextEdit()
+        self.txt_log.setReadOnly(True)
+        self.txt_log.setStyleSheet("font-family: monospace; font-size: 11px;")
+
+        layout.addWidget(self.lbl_title)
+        layout.addWidget(self.txt_log)
+
+    def append_log(self, message):
+        self.txt_log.append(message)
+
+    def clear_logs(self):
+        self.txt_log.clear()

src/gui/main_window.py

@@ -0,0 +1,51 @@
+# src/gui/main_window.py
+
+from PySide6.QtWidgets import QMainWindow, QTabWidget
+from gui.tabs.extraction_tab import ExtractionTab
+from gui.tabs.database_tab import DatabaseTab
+from gui.tabs.settings_tab import SettingsTab
+from services.sync_worker import SyncWorker
+
+class MainWindow(QMainWindow):
+    def __init__(self):
+        super().__init__()
+        self.setWindowTitle("144-python-email-extractor")
+        self.resize(1000, 700)
+
+        # Tab Widget Container
+        self.tabs = QTabWidget()
+        self.setCentralWidget(self.tabs)
+
+        # Instantiate Tabs
+        self.tab_extraction = ExtractionTab(start_sync_callback=self.run_sync_process)
+        self.tab_database = DatabaseTab()
+        self.tab_settings = SettingsTab()
+
+        # Add pages to Tab Control
+        self.tabs.addTab(self.tab_extraction, "Extract & Sync")
+        self.tabs.addTab(self.tab_database, "Database Inspector")
+        self.tabs.addTab(self.tab_settings, "Account Settings")
+
+    def run_sync_process(self, project_name, organization):
+        self.tab_extraction.set_syncing_state(True)
+        self.tab_extraction.log_viewer.clear_logs()
+
+        ms_creds = self.tab_settings.get_ms_credentials()
+        gmail_file = self.tab_settings.get_gmail_credentials_file()
+
+        # Launch non-blocking QThread worker
+        self.worker = SyncWorker(
+            project_name=project_name,
+            organization=organization,
+            ms_credentials=ms_creds,
+            gmail_credentials_file=gmail_file
+        )
+        
+        self.worker.log_signal.connect(self.tab_extraction.log_viewer.append_log)
+        self.worker.finished_signal.connect(self.on_sync_complete)
+        self.worker.start()
+
+    def on_sync_complete(self):
+        self.tab_extraction.set_syncing_state(False)
+        self.tab_database.refresh_table()
+        

src/gui/tabs/database_tab.py

@@ -0,0 +1,72 @@
+# src/gui/tabs/database_tab.py
+
+from PySide6.QtWidgets import (
+    QWidget, QVBoxLayout, QHBoxLayout, QTableWidget, QTableWidgetItem, 
+    QHeaderView, QLabel, QTextEdit, QSplitter, QPushButton
+)
+from PySide6.QtCore import Qt
+from models.database import fetch_all_emails
+
+class DatabaseTab(QWidget):
+    def __init__(self, parent=None):
+        super().__init__(parent)
+        self.init_ui()
+
+    def init_ui(self):
+        layout = QVBoxLayout(self)
+
+        # Top Control Bar
+        top_bar = QHBoxLayout()
+        self.btn_refresh = QPushButton("Refresh Table")
+        self.btn_refresh.clicked.connect(self.refresh_table)
+        top_bar.addWidget(QLabel("SQLite Indexed Records:"))
+        top_bar.addStretch()
+        top_bar.addWidget(self.btn_refresh)
+        layout.addLayout(top_bar)
+
+        # Main Splitter (Table on top/left, JSON detail on bottom/right)
+        splitter = QSplitter(Qt.Vertical)
+
+        self.table = QTableWidget()
+        self.table.setColumnCount(6)
+        self.table.setHorizontalHeaderLabels(["ID", "Account", "Subject", "Sender", "Date", "File Path"])
+        self.table.horizontalHeader().setSectionResizeMode(QHeaderView.ResizeToContents)
+        self.table.horizontalHeader().setSectionResizeMode(2, QHeaderView.Stretch)
+        self.table.itemSelectionChanged.connect(self.on_row_selected)
+        splitter.addWidget(self.table)
+
+        # JSON Inspector Box
+        json_box = QWidget()
+        box_layout = QVBoxLayout(json_box)
+        box_layout.setContentsMargins(0, 0, 0, 0)
+        box_layout.addWidget(QLabel("JSON Metadata Preview:"))
+        self.txt_json = QTextEdit()
+        self.txt_json.setReadOnly(True)
+        self.txt_json.setStyleSheet("font-family: monospace;")
+        box_layout.addWidget(self.txt_json)
+
+        splitter.addWidget(json_box)
+        splitter.setSizes([350, 150])
+
+        layout.addWidget(splitter)
+        self.refresh_table()
+
+    def refresh_table(self):
+        rows = fetch_all_emails()
+        self.table.setRowCount(0)
+        self.raw_rows = rows  # Keep full data in memory for JSON preview
+        
+        for row_idx, row_data in enumerate(rows):
+            self.table.insertRow(row_idx)
+            # Render ID, Account, Subject, Sender, Date, Path
+            for col_idx in range(6):
+                val = str(row_data[col_idx])
+                item = QTableWidgetItem(val)
+                self.table.setItem(row_idx, col_idx, item)
+
+    def on_row_selected(self):
+        curr_row = self.table.currentRow()
+        if curr_row >= 0 and curr_row < len(self.raw_rows):
+            # Extra JSON metadata is at index 6 in the tuple
+            json_blob = self.raw_rows[curr_row][6]
+            self.txt_json.setText(json_blob or "{}")

src/gui/tabs/extraction_tab.py

@@ -0,0 +1,46 @@
+# src/gui/tabs/extraction_tab.py
+from PySide6.QtWidgets import (
+    QWidget, QVBoxLayout, QHBoxLayout, QGroupBox, QLineEdit, QPushButton, QLabel
+)
+from gui.components.log_viewer import LogViewer
+
+class ExtractionTab(QWidget):
+    def __init__(self, start_sync_callback, parent=None):
+        super().__init__(parent)
+        self.start_sync_callback = start_sync_callback
+        self.init_ui()
+
+    def init_ui(self):
+        layout = QVBoxLayout(self)
+
+        # Metadata Configuration Parameters
+        config_box = QGroupBox("Extraction Metadata Setup")
+        cfg_layout = QHBoxLayout(config_box)
+
+        self.txt_project = QLineEdit("Project_144")
+        self.txt_project.setPlaceholderText("Project Name")
+        self.txt_org = QLineEdit("Oxnee")
+        self.txt_org.setPlaceholderText("Organization")
+
+        self.btn_sync = QPushButton("Start Download & Extraction")
+        self.btn_sync.clicked.connect(self.on_sync_clicked)
+
+        cfg_layout.addWidget(QLabel("Project:"))
+        cfg_layout.addWidget(self.txt_project)
+        cfg_layout.addWidget(QLabel("Organization:"))
+        cfg_layout.addWidget(self.txt_org)
+        cfg_layout.addWidget(self.btn_sync)
+
+        layout.addWidget(config_box)
+
+        # Log Viewer Component
+        self.log_viewer = LogViewer()
+        layout.addWidget(self.log_viewer)
+
+    def on_sync_clicked(self):
+        proj = self.txt_project.text().strip()
+        org = self.txt_org.text().strip()
+        self.start_sync_callback(proj, org)
+
+    def set_syncing_state(self, is_syncing):
+        self.btn_sync.setEnabled(not is_syncing)

src/main.py

@@ -0,0 +1,23 @@
+# src/main.py
+
+import sys
+import os
+
+# Ensure src path resolution when running via uv
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+
+from PySide6.QtWidgets import QApplication
+from gui.main_window import MainWindow
+from models.database import init_db
+
+def main():
+    init_db()  # Ensure database schema is ready
+    app = QApplication(sys.argv)
+    
+    window = MainWindow()
+    window.show()
+    
+    sys.exit(app.exec())
+
+if __name__ == "__main__":
+    main()

src/models/database.py

@@ -0,0 +1,59 @@
+# src/models/database.py
+
+import sqlite3
+import json
+import os
+
+DB_FILE = "email_archive.db"
+
+def init_db(db_path=DB_FILE):
+    """Initializes the SQLite database schema if it doesn't exist."""
+    conn = sqlite3.connect(db_path)
+    cursor = conn.cursor()
+    
+    cursor.execute("""
+    CREATE TABLE IF NOT EXISTS emails (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        account_email TEXT NOT NULL,
+        message_id TEXT NOT NULL UNIQUE,
+        subject TEXT,
+        sender TEXT,
+        received_date TEXT,
+        file_path TEXT NOT NULL,
+        extra_metadata TEXT, -- JSON blob for flexible fields (project, org, tags, etc.)
+        created_at DATETIME DEFAULT CURRENT_TIMESTAMP
+    )
+    """)
+    conn.commit()
+    conn.close()
+
+def save_email_record(account_email, message_id, subject, sender, received_date, file_path, metadata_dict, db_path=DB_FILE):
+    """Inserts or updates an email record with extra JSON metadata."""
+    conn = sqlite3.connect(db_path)
+    cursor = conn.cursor()
+    
+    json_metadata = json.dumps(metadata_dict, indent=2)
+    
+    cursor.execute("""
+    INSERT INTO emails (account_email, message_id, subject, sender, received_date, file_path, extra_metadata)
+    VALUES (?, ?, ?, ?, ?, ?, ?)
+    ON CONFLICT(message_id) DO UPDATE SET
+        extra_metadata = excluded.extra_metadata,
+        file_path = excluded.file_path,
+        subject = excluded.subject
+    """, (account_email, message_id, subject, sender, received_date, file_path, json_metadata))
+    
+    conn.commit()
+    conn.close()
+
+def fetch_all_emails(db_path=DB_FILE):
+    """Retrieves all indexed email records ordered by newest ID."""
+    conn = sqlite3.connect(db_path)
+    cursor = conn.cursor()
+    cursor.execute("""
+        SELECT id, account_email, subject, sender, received_date, file_path, extra_metadata 
+        FROM emails ORDER BY id DESC
+    """)
+    rows = cursor.fetchall()
+    conn.close()
+    return rows

src/services/gmail_api.py

@@ -0,0 +1,84 @@
+# src/services/gmail_api.py
+import os
+import re
+import base64
+from google_auth_oauthlib.flow import InstalledAppFlow
+from googleapiclient.discovery import build
+from google.auth.transport.requests import Request
+from google.oauth2.credentials import Credentials
+
+STORAGE_DIR = os.path.abspath("./storage")
+
+def sanitize_filename(name):
+    clean = re.sub(r'[\\/*?:"<>|]', "", name or "Untitled")
+    return clean[:80].strip()
+
+class GmailDownloader:
+    SCOPES = ['https://www.googleapis.com/auth/gmail.readonly']
+
+    def __init__(self, credentials_json="credentials.json", account_email="slohning@gmail.com"):
+        self.account_email = account_email
+        self.creds_json = credentials_json
+        self.service = self._authenticate()
+
+    def _authenticate(self):
+        if not os.path.exists(self.creds_json):
+            return None
+
+        creds = None
+        token_file = 'token_gmail.json'
+        if os.path.exists(token_file):
+            creds = Credentials.from_authorized_user_file(token_file, self.SCOPES)
+        if not creds or not creds.valid:
+            if creds and creds.expired and creds.refresh_token:
+                creds.refresh(Request())
+            else:
+                flow = InstalledAppFlow.from_client_secrets_file(self.creds_json, self.SCOPES)
+                creds = flow.run_local_server(port=0)
+            with open(token_file, 'w') as token:
+                token.write(creds.to_json())
+        return build('gmail', 'v1', credentials=creds)
+
+    def download_inbox(self, callback_signal):
+        if not self.service:
+            callback_signal.emit(f"[{self.account_email}] Auth Error: Missing {self.creds_json} configuration.")
+            return []
+
+        results = self.service.users().messages().list(userId='me', maxResults=20, q="label:INBOX").execute()
+        messages = results.get('messages', [])
+        
+        extracted = []
+        os.makedirs(STORAGE_DIR, exist_ok=True)
+
+        for m in messages:
+            msg_id = m['id']
+            # Fetch raw RFC 822 format
+            raw_msg = self.service.users().messages().get(userId='me', id=msg_id, format='raw').execute()
+            
+            meta_msg = self.service.users().messages().get(userId='me', id=msg_id, format='full').execute()
+            headers = meta_msg.get('payload', {}).get('headers', [])
+            
+            subject = next((h['value'] for h in headers if h['name'].lower() == 'subject'), 'No Subject')
+            sender = next((h['value'] for h in headers if h['name'].lower() == 'from'), 'Unknown')
+            date_str = next((h['value'] for h in headers if h['name'].lower() == 'date'), 'UnknownDate')
+
+            eml_data = base64.urlsafe_b64decode(raw_msg['raw'].encode('ASCII'))
+            
+            fname = f"GMAIL_{sanitize_filename(subject)}_{msg_id[-6:]}.eml"
+            fpath = os.path.abspath(os.path.join(STORAGE_DIR, fname))
+            
+            with open(fpath, "wb") as f:
+                f.write(eml_data)
+                
+            extracted.append({
+                "account": self.account_email,
+                "id": msg_id,
+                "subject": subject,
+                "sender": sender,
+                "date": date_str,
+                "path": fpath
+            })
+            callback_signal.emit(f"[{self.account_email}] Downloaded: {fname}")
+            
+        return extracted
+    

src/services/ms_graph.py

@@ -0,0 +1,80 @@
+# src/services/ms_graph.py
+import os
+import re
+import requests
+import msal
+
+STORAGE_DIR = os.path.abspath("./storage")
+
+def sanitize_filename(name):
+    clean = re.sub(r'[\\/*?:"<>|]', "", name or "Untitled")
+    return clean[:80].strip()
+
+class MSGraphDownloader:
+    def __init__(self, tenant_id, client_id, client_secret, account_email):
+        self.account_email = account_email
+        self.tenant_id = tenant_id
+        self.client_id = client_id
+        self.client_secret = client_secret
+        self.token = self._get_token()
+
+    def _get_token(self):
+        if not self.client_id or not self.client_secret or not self.tenant_id:
+            return None
+            
+        app = msal.ConfidentialClientApplication(
+            self.client_id,
+            authority=f"https://login.microsoftonline.com/{self.tenant_id}",
+            client_credential=self.client_secret,
+        )
+        res = app.acquire_token_for_client(scopes=["https://graph.microsoft.com/.default"])
+        return res.get("access_token")
+
+    def download_inbox(self, callback_signal):
+        if not self.token:
+            callback_signal.emit(f"[{self.account_email}] Auth Error: Missing or invalid MS Azure credentials.")
+            return []
+
+        headers = {"Authorization": f"Bearer {self.token}", "Accept": "application/json"}
+        url = f"https://graph.microsoft.com/v1.0/users/{self.account_email}/mailFolders/inbox/messages?$select=id,subject,from,receivedDateTime&$top=20"
+        
+        extracted = []
+        os.makedirs(STORAGE_DIR, exist_ok=True)
+
+        res = requests.get(url, headers=headers)
+        if res.status_code != 200:
+            callback_signal.emit(f"[{self.account_email}] Failed to fetch messages: Status {res.status_code}")
+            return []
+
+        messages = res.json().get("value", [])
+        for msg in messages:
+            msg_id = msg["id"]
+            subject = msg.get("subject", "No Subject")
+            sender = msg.get("from", {}).get("emailAddress", {}).get("address", "Unknown")
+            date_str = msg.get("receivedDateTime", "")[:10]
+            
+            # Request raw MIME payload
+            raw_url = f"https://graph.microsoft.com/v1.0/users/{self.account_email}/messages/{msg_id}/$value"
+            eml_res = requests.get(raw_url, headers=headers)
+            
+            if eml_res.status_code == 200:
+                fname = f"MS_{date_str}_{sanitize_filename(subject)}_{msg_id[-6:]}.eml"
+                fpath = os.path.abspath(os.path.join(STORAGE_DIR, fname))
+                
+                with open(fpath, "wb") as f:
+                    f.write(eml_res.content)
+                
+                extracted.append({
+                    "account": self.account_email,
+                    "id": msg_id,
+                    "subject": subject,
+                    "sender": sender,
+                    "date": date_str,
+                    "path": fpath
+                })
+                callback_signal.emit(f"[{self.account_email}] Downloaded: {fname}")
+            else:
+                callback_signal.emit(f"[{self.account_email}] Failed raw fetch for {msg_id[-6:]}")
+                
+        return extracted
+    

src/services/sync_worker.py

@@ -0,0 +1,65 @@
+# src/services/sync_worker.py
+
+from PySide6.QtCore import QThread, Signal
+from models.database import init_db, save_email_record
+from services.ms_graph import MSGraphDownloader
+from services.gmail_api import GmailDownloader
+
+class SyncWorker(QThread):
+    log_signal = Signal(str)
+    finished_signal = Signal()
+
+    def __init__(self, project_name, organization, ms_credentials, gmail_credentials_file):
+        super().__init__()
+        self.project_name = project_name
+        self.organization = organization
+        self.ms_credentials = ms_credentials
+        self.gmail_credentials_file = gmail_credentials_file
+
+    def run(self):
+        self.log_signal.emit("Initializing database and network drivers...")
+        init_db()
+
+        records = []
+
+        # 1. Sync Microsoft Accounts
+        ms_accounts = ["stephen.lohning@oxnee.com", "slohning@live.com.au"]
+        for email in ms_accounts:
+            self.log_signal.emit(f"Connecting to Microsoft Graph for {email}...")
+            ms = MSGraphDownloader(
+                tenant_id=self.ms_credentials.get("tenant_id"),
+                client_id=self.ms_credentials.get("client_id"),
+                client_secret=self.ms_credentials.get("client_secret"),
+                account_email=email
+            )
+            records.extend(ms.download_inbox(self.log_signal))
+
+        # 2. Sync Gmail Account
+        self.log_signal.emit("Connecting to Google Gmail API for slohning@gmail.com...")
+        try:
+            gm = GmailDownloader(self.gmail_credentials_file, "slohning@gmail.com")
+            records.extend(gm.download_inbox(self.log_signal))
+        except Exception as e:
+            self.log_signal.emit(f"[slohning@gmail.com] Sync Error: {e}")
+
+        # 3. Save Records and metadata JSON into SQLite
+        self.log_signal.emit(f"Indexing {len(records)} files into SQLite database...")
+        for r in records:
+            extra_meta = {
+                "project_name": self.project_name,
+                "organization": self.organization,
+                "app_name": "144-python-email-extractor",
+                "version": "1.0.0"
+            }
+            save_email_record(
+                account_email=r["account"],
+                message_id=r["id"],
+                subject=r["subject"],
+                sender=r["sender"],
+                received_date=r["date"],
+                file_path=r["path"],
+                metadata_dict=extra_meta
+            )
+
+        self.log_signal.emit("\nExtraction and database indexing finished successfully!")
+        self.finished_signal.emit()