initial

app/content/extractors.py

@@ -0,0 +1,62 @@
+from pathlib import Path
+from typing import Dict, Optional
+import json
+
+class ContentExtractor:
+
+    def __init__(self):
+        self.extractors = {'pdf_text': self._extract_pdf, 'ocr+caption': self._extract_image, 'transcribe': self._extract_audio, 'transcribe+scenes': self._extract_video, 'office_text': self._extract_document, 'read': self._extract_text, 'read+syntax': self._extract_code}
+
+    def extract(self, file_path: Path, extractor_type: str) -> Dict:
+        extractor = self.extractors.get(extractor_type)
+        if not extractor:
+            return {'error': f'Unknown extractor: {extractor_type}'}
+        try:
+            return extractor(file_path)
+        except Exception as e:
+            return {'error': str(e)}
+
+    def _extract_text(self, file_path: Path) -> Dict:
+        try:
+            with open(file_path, 'r', encoding='utf-8', errors='ignore') as f:
+                content = f.read(1024 * 1024)
+            return {'text': content, 'char_count': len(content), 'needs_llm': False}
+        except Exception as e:
+            return {'error': str(e)}
+
+    def _extract_code(self, file_path: Path) -> Dict:
+        result = self._extract_text(file_path)
+        if 'error' not in result:
+            result['type'] = 'code'
+            result['needs_llm'] = True
+        return result
+
+    def _extract_pdf(self, file_path: Path) -> Dict:
+        try:
+            import PyPDF2
+            text_parts = []
+            with open(file_path, 'rb') as f:
+                pdf = PyPDF2.PdfReader(f)
+                for page in pdf.pages[:10]:
+                    text_parts.append(page.extract_text())
+            text = '\n'.join(text_parts)
+            return {'text': text, 'pages_extracted': len(text_parts), 'needs_llm': len(text.strip()) > 100, 'type': 'document'}
+        except Exception as e:
+            return {'error': str(e), 'needs_ocr': True}
+
+    def _extract_image(self, file_path: Path) -> Dict:
+        return {'type': 'image', 'needs_ocr': True, 'needs_caption': True, 'needs_llm': True, 'pipeline': ['ocr', 'caption', 'embedding'], 'status': 'pending'}
+
+    def _extract_audio(self, file_path: Path) -> Dict:
+        return {'type': 'audio', 'needs_transcription': True, 'needs_llm': True, 'pipeline': ['transcribe', 'summarize'], 'status': 'pending'}
+
+    def _extract_video(self, file_path: Path) -> Dict:
+        return {'type': 'video', 'needs_transcription': True, 'needs_scene_detection': True, 'needs_llm': True, 'pipeline': ['transcribe', 'scenes', 'summarize'], 'status': 'pending'}
+
+    def _extract_document(self, file_path: Path) -> Dict:
+        try:
+            import textract
+            text = textract.process(str(file_path)).decode('utf-8')
+            return {'text': text, 'type': 'document', 'needs_llm': len(text.strip()) > 100}
+        except:
+            return {'error': 'textract failed', 'needs_llm': True}