generated timeline structure and stats

scripts/overview/data-overview.py

@@ -178,6 +178,134 @@ def analyze_timeline_data(timeline_path: str) -> Dict[str, Any]:
     
     return stats
 
+def export_statistics_to_file(stats: Dict[str, Any], output_path: str):
+    """Export comprehensive statistics to a text file."""
+    with open(output_path, 'w', encoding='utf-8') as f:
+        # Redirect print statements to file
+        original_stdout = sys.stdout
+        sys.stdout = f
+        
+        print_statistics(stats)
+        
+        # Restore stdout
+        sys.stdout = original_stdout
+
+def analyze_json_structure(timeline_path: str, output_path: str):
+    """Analyze and export the JSON structure to a text file."""
+    print(f"Analyzing JSON structure from: {timeline_path}")
+    
+    with open(timeline_path, 'r', encoding='utf-8') as f:
+        data = json.load(f)
+    
+    def explore_structure(obj, path="", depth=0, max_depth=4):
+        """Recursively explore JSON structure."""
+        indent = "  " * depth
+        structure_info = []
+        
+        if depth > max_depth:
+            return [f"{indent}... (max depth reached)"]
+        
+        if isinstance(obj, dict):
+            structure_info.append(f"{indent}{path} (dict) - {len(obj)} keys:")
+            for key, value in list(obj.items())[:10]:  # Limit to first 10 keys
+                key_path = f"{path}.{key}" if path else key
+                if isinstance(value, (dict, list)):
+                    structure_info.extend(explore_structure(value, key_path, depth + 1, max_depth))
+                else:
+                    value_type = type(value).__name__
+                    if isinstance(value, str) and len(value) > 50:
+                        sample = value[:50] + "..."
+                    else:
+                        sample = str(value)
+                    structure_info.append(f"{indent}  {key}: {value_type} = {sample}")
+            
+            if len(obj) > 10:
+                structure_info.append(f"{indent}  ... and {len(obj) - 10} more keys")
+                
+        elif isinstance(obj, list):
+            structure_info.append(f"{indent}{path} (list) - {len(obj)} items:")
+            if obj:
+                structure_info.append(f"{indent}  Sample item structure:")
+                structure_info.extend(explore_structure(obj[0], f"{path}[0]", depth + 1, max_depth))
+                if len(obj) > 1:
+                    structure_info.append(f"{indent}  ... and {len(obj) - 1} more items")
+        else:
+            value_type = type(obj).__name__
+            structure_info.append(f"{indent}{path}: {value_type} = {obj}")
+        
+        return structure_info
+    
+    with open(output_path, 'w', encoding='utf-8') as f:
+        f.write("="*80 + "\n")
+        f.write("TIMELINE JSON STRUCTURE ANALYSIS\n")
+        f.write("="*80 + "\n\n")
+        
+        f.write(f"File: {timeline_path}\n")
+        f.write(f"Analysis Date: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n\n")
+        
+        # Overall structure
+        f.write("ROOT LEVEL STRUCTURE:\n")
+        f.write("-" * 40 + "\n")
+        for key, value in data.items():
+            if isinstance(value, list):
+                f.write(f"{key}: list with {len(value)} items\n")
+            elif isinstance(value, dict):
+                f.write(f"{key}: dict with {len(value)} keys\n")
+            else:
+                f.write(f"{key}: {type(value).__name__} = {value}\n")
+        
+        f.write("\n" + "="*80 + "\n")
+        f.write("DETAILED STRUCTURE:\n")
+        f.write("="*80 + "\n\n")
+        
+        # Detailed structure analysis
+        structure_lines = explore_structure(data)
+        for line in structure_lines:
+            f.write(line + "\n")
+        
+        # Sample semantic segment analysis
+        semantic_segments = data.get('semanticSegments', [])
+        if semantic_segments:
+            f.write("\n" + "="*80 + "\n")
+            f.write("SEMANTIC SEGMENTS ANALYSIS:\n")
+            f.write("="*80 + "\n\n")
+            
+            f.write(f"Total semantic segments: {len(semantic_segments)}\n\n")
+            
+            # Analyze different types of segments
+            visit_count = sum(1 for seg in semantic_segments if 'visit' in seg)
+            path_count = sum(1 for seg in semantic_segments if 'timelinePath' in seg)
+            
+            f.write(f"Segments with visits: {visit_count}\n")
+            f.write(f"Segments with timeline paths: {path_count}\n\n")
+            
+            # Sample visit structure
+            sample_visit = None
+            sample_path = None
+            
+            for segment in semantic_segments[:100]:  # Check first 100 segments
+                if 'visit' in segment and sample_visit is None:
+                    sample_visit = segment
+                if 'timelinePath' in segment and sample_path is None:
+                    sample_path = segment
+                if sample_visit and sample_path:
+                    break
+            
+            if sample_visit:
+                f.write("SAMPLE VISIT STRUCTURE:\n")
+                f.write("-" * 40 + "\n")
+                visit_structure = explore_structure(sample_visit, "sample_visit")
+                for line in visit_structure:
+                    f.write(line + "\n")
+                f.write("\n")
+            
+            if sample_path:
+                f.write("SAMPLE TIMELINE PATH STRUCTURE:\n")
+                f.write("-" * 40 + "\n")
+                path_structure = explore_structure(sample_path, "sample_timelinePath")
+                for line in path_structure:
+                    f.write(line + "\n")
+
 def print_statistics(stats: Dict[str, Any]):
     """Print comprehensive statistics in a readable format."""
     print("\n" + "="*80)
@@ -277,14 +405,32 @@ def main():
         print(f"Searched in: {repo_root}")
         sys.exit(1)
     
+    # Generate output file names with timestamp
+    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+    stats_output_path = os.path.join(repo_root, f"timeline_statistics_{timestamp}.txt")
+    structure_output_path = os.path.join(repo_root, f"timeline_structure_{timestamp}.txt")
+    
     try:
+        # Analyze the JSON structure first
+        print("📋 Analyzing JSON structure...")
+        analyze_json_structure(timeline_path, structure_output_path)
+        print(f"✅ JSON structure exported to: {structure_output_path}")
+        
         # Analyze the data
         stats = analyze_timeline_data(timeline_path)
         
-        # Print the results
+        # Print the results to console
         print_statistics(stats)
         
-        print(f"\n✅ Analysis complete! File analyzed: {timeline_path}")
+        # Export statistics to file
+        print(f"\n📄 Exporting statistics to file...")
+        export_statistics_to_file(stats, stats_output_path)
+        print(f"✅ Statistics exported to: {stats_output_path}")
+        
+        print(f"\n🎉 Analysis complete!")
+        print(f"📊 Statistics file: {stats_output_path}")
+        print(f"🏗️  Structure file: {structure_output_path}")
+        print(f"📁 Source file: {timeline_path}")
         
     except FileNotFoundError:
         print(f"❌ Error: Could not find Timeline.json at {timeline_path}")

scripts/overview/timeline_statistics_20250831_163518.txt

@@ -0,0 +1,84 @@
+
+================================================================================
+TIMELINE DATA OVERVIEW
+================================================================================
+
+📊 BASIC STATISTICS
+   Total segments: 43,295
+   Visits: 14,890
+   Timeline paths: 13,322
+   Unique places: 2,766
+
+📅 DATE RANGE
+   Earliest record: 2013-12-31 22:00:00
+   Latest record: 2025-08-31 04:00:00
+   Total span: 4,260 days (11.7 years)
+
+⏰ DURATION STATISTICS
+   Total tracked time: 95,930.7 hours
+   Average per day: 22.5 hours
+
+🎯 PROBABILITY STATISTICS
+   Average confidence: 0.871
+   Min confidence: 0.100
+   Max confidence: 1.000
+
+🌍 GEOGRAPHIC STATISTICS
+   Northern bound: 56.375116°
+   Southern bound: 15.327795°
+   Eastern bound: 73.898877°
+   Western bound: -81.581212°
+   Max distance span: 14483.9 km
+
+🏷️  TOP LOCATION TYPES
+   UNKNOWN        : 9,398 (63.1%)
+   HOME           : 2,521 (16.9%)
+   ALIASED_LOCATION: 1,093 (7.3%)
+   SEARCHED_ADDRESS: 875 (5.9%)
+   WORK           : 555 (3.7%)
+   INFERRED_HOME  : 237 (1.6%)
+   INFERRED_WORK  : 211 (1.4%)
+
+📍 TOP VISITED PLACES
+   #1  ChIJ4wTqNJRqdkgRjhi7lioZa2Y   : 1,587 visits (10.7%)
+   #2  ChIJyaJWtZVqdkgRZHVIi0HKLto   : 893 visits (6.0%)
+   #3  ChIJfyzFaC4bdkgRSZJuOgfp2iQ   : 598 visits (4.0%)
+   #4  ChIJdSS3YTobdkgR5kspQSjORnU   : 570 visits (3.8%)
+   #5  ChIJNfV396VrdkgRV3_rnaQyxl8   : 369 visits (2.5%)
+
+📈 ACTIVITY BY YEAR
+   2013: 2 segments
+   2014: 212 segments
+   2015: 627 segments
+   2016: 21 segments
+   2018: 2,165 segments
+   2019: 5,071 segments
+   2020: 2,441 segments
+   2021: 5,874 segments
+   2022: 7,360 segments
+   2023: 7,296 segments
+   2024: 7,379 segments
+   2025: 4,847 segments
+
+📆 ACTIVITY BY DAY OF WEEK
+   Monday    : 6,076 segments
+   Tuesday   : 6,170 segments
+   Wednesday : 6,177 segments
+   Thursday  : 6,386 segments
+   Friday    : 6,869 segments
+   Saturday  : 6,328 segments
+   Sunday    : 5,289 segments
+
+📊 RECENT MONTHLY ACTIVITY
+   2024-09: 557 segments
+   2024-10: 519 segments
+   2024-11: 605 segments
+   2024-12: 683 segments
+   2025-01: 602 segments
+   2025-02: 572 segments
+   2025-03: 588 segments
+   2025-04: 663 segments
+   2025-05: 631 segments
+   2025-06: 593 segments
+   2025-07: 642 segments
+   2025-08: 556 segments

scripts/overview/timeline_structure_20250831_163518.txt

@@ -0,0 +1,109 @@
+================================================================================
+TIMELINE JSON STRUCTURE ANALYSIS
+================================================================================
+
+File: /Users/azeem/repos/personal-tracker/data/Timeline.json
+Analysis Date: 2025-08-31 16:35:18
+
+ROOT LEVEL STRUCTURE:
+----------------------------------------
+semanticSegments: list with 43295 items
+rawSignals: list with 9549 items
+userLocationProfile: dict with 3 keys
+
+================================================================================
+DETAILED STRUCTURE:
+================================================================================
+
+ (dict) - 3 keys:
+  semanticSegments (list) - 43295 items:
+    Sample item structure:
+    semanticSegments[0] (dict) - 3 keys:
+      startTime: str = 2013-12-31T22:00:00.000+00:00
+      endTime: str = 2014-01-01T00:00:00.000+00:00
+      semanticSegments[0].timelinePath (list) - 1 items:
+        Sample item structure:
+        semanticSegments[0].timelinePath[0] (dict) - 2 keys:
+          point: str = 51.6659027°, -0.4058773°
+          time: str = 2013-12-31T22:29:00.000+00:00
+    ... and 43294 more items
+  rawSignals (list) - 9549 items:
+    Sample item structure:
+    rawSignals[0] (dict) - 1 keys:
+      rawSignals[0].position (dict) - 6 keys:
+        LatLng: str = 51.5151973°, -0.1312666°
+        accuracyMeters: int = 100
+        altitudeMeters: float = 86.30000305175781
+        source: str = UNKNOWN
+        timestamp: str = 2025-08-01T16:09:32.000+01:00
+        speedMetersPerSecond: float = 0.0
+    ... and 9548 more items
+  userLocationProfile (dict) - 3 keys:
+    userLocationProfile.frequentPlaces (list) - 10 items:
+      Sample item structure:
+      userLocationProfile.frequentPlaces[0] (dict) - 3 keys:
+        placeId: str = ChIJAAAAAAAAAAARjhi7lioZa2Y
+        placeLocation: str = 51.6658192°, -0.4056977°
+        label: str = HOME
+      ... and 9 more items
+    userLocationProfile.frequentTrips (list) - 9 items:
+      Sample item structure:
+      userLocationProfile.frequentTrips[0] (dict) - 7 keys:
+        userLocationProfile.frequentTrips[0].waypointIds (list) - 4 items:
+          Sample item structure:
+          ... (max depth reached)
+          ... and 3 more items
+        userLocationProfile.frequentTrips[0].modeDistribution (list) - 4 items:
+          Sample item structure:
+          ... (max depth reached)
+          ... and 3 more items
+        startTimeMinutes: int = 504
+        endTimeMinutes: int = 551
+        durationMinutes: int = 47
+        confidence: float = 0.0
+        commuteDirection: str = COMMUTE_DIRECTION_HOME_TO_WORK
+      ... and 8 more items
+    userLocationProfile.persona (dict) - 1 keys:
+      userLocationProfile.persona.travelModeAffinities (list) - 7 items:
+        Sample item structure:
+        userLocationProfile.persona.travelModeAffinities[0] (dict) - 2 keys:
+          mode: str = WALKING
+          affinity: float = 0.4837758243083954
+        ... and 6 more items
+
+================================================================================
+SEMANTIC SEGMENTS ANALYSIS:
+================================================================================
+
+Total semantic segments: 43295
+
+Segments with visits: 14890
+Segments with timeline paths: 13322
+
+SAMPLE VISIT STRUCTURE:
+----------------------------------------
+sample_visit (dict) - 5 keys:
+  startTime: str = 2013-12-31T22:29:21.000+00:00
+  endTime: str = 2014-01-01T17:10:30.000+00:00
+  startTimeTimezoneUtcOffsetMinutes: int = 0
+  endTimeTimezoneUtcOffsetMinutes: int = 0
+  sample_visit.visit (dict) - 3 keys:
+    hierarchyLevel: int = 0
+    probability: float = 0.6399999856948853
+    sample_visit.visit.topCandidate (dict) - 4 keys:
+      placeId: str = ChIJyaJWtZVqdkgRZHVIi0HKLto
+      semanticType: str = HOME
+      probability: float = 0.9986714720726013
+      sample_visit.visit.topCandidate.placeLocation (dict) - 1 keys:
+        latLng: str = 51.6659399°, -0.4059464°
+
+SAMPLE TIMELINE PATH STRUCTURE:
+----------------------------------------
+sample_timelinePath (dict) - 3 keys:
+  startTime: str = 2013-12-31T22:00:00.000+00:00
+  endTime: str = 2014-01-01T00:00:00.000+00:00
+  sample_timelinePath.timelinePath (list) - 1 items:
+    Sample item structure:
+    sample_timelinePath.timelinePath[0] (dict) - 2 keys:
+      point: str = 51.6659027°, -0.4058773°
+      time: str = 2013-12-31T22:29:00.000+00:00