Code sketch. Download cache working.
This commit is contained in:
21
.cspell/custom-dictionary-workspace.txt
Normal file
21
.cspell/custom-dictionary-workspace.txt
Normal file
@@ -0,0 +1,21 @@
|
||||
# Custom Dictionary Words
|
||||
Arrington
|
||||
bioguide
|
||||
dotenv
|
||||
gitlaw
|
||||
HRES
|
||||
Jodey
|
||||
Lauro
|
||||
levelname
|
||||
oneline
|
||||
pathlib
|
||||
Pelosi
|
||||
Pydantic
|
||||
pyproject
|
||||
pytest
|
||||
relatedbills
|
||||
Schumer
|
||||
SRES
|
||||
usc
|
||||
uscode
|
||||
Yarmuth
|
||||
53
.gitignore
vendored
Normal file
53
.gitignore
vendored
Normal file
@@ -0,0 +1,53 @@
|
||||
# Environment and API keys
|
||||
.env
|
||||
|
||||
# Python
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
*.so
|
||||
.Python
|
||||
build/
|
||||
dist/
|
||||
wheels/
|
||||
*.egg-info
|
||||
.venv/
|
||||
env/
|
||||
venv/
|
||||
ENV/
|
||||
env.bak/
|
||||
venv.bak/
|
||||
|
||||
# Package managers
|
||||
uv.lock
|
||||
pip-*.txt
|
||||
|
||||
# IDEs
|
||||
.vscode/
|
||||
.idea/
|
||||
*.swp
|
||||
*.swo
|
||||
.ruff_cache/
|
||||
|
||||
|
||||
# OS
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
.directory
|
||||
|
||||
# Logs
|
||||
*.log
|
||||
logs/
|
||||
|
||||
# Temporary files
|
||||
temp/
|
||||
tmp/
|
||||
.tmp/
|
||||
|
||||
data/
|
||||
logs/
|
||||
uscode-git-datastore/
|
||||
uscode-git-blame/
|
||||
download_cache/
|
||||
download_cache
|
||||
cache/
|
||||
6
.gitmodules
vendored
Normal file
6
.gitmodules
vendored
Normal file
@@ -0,0 +1,6 @@
|
||||
[submodule "bill-dtd"]
|
||||
path = bill-dtd
|
||||
url = https://github.com/usgpo/bill-dtd.git
|
||||
[submodule "uslm"]
|
||||
path = uslm
|
||||
url = https://github.com/usgpo/uslm.git
|
||||
535
README.md
535
README.md
@@ -1,3 +1,532 @@
|
||||
# git-law
|
||||
|
||||
Git Blame for the United States Code
|
||||
# 🏛️ Git Blame for the United States Code
|
||||
|
||||
> **Apply the full power of git to track every change in the United States Code with line-by-line attribution to Congressional sponsors.**
|
||||
|
||||
[](https://www.python.org/downloads/)
|
||||
[](https://pydantic.dev/)
|
||||
[](https://api.congress.gov/)
|
||||
|
||||
## Vision: True Git Blame for Law
|
||||
|
||||
```bash
|
||||
git blame Title-42-The-Public-Health-and-Welfare/Chapter-06A-Public-Health-Service/Section-280g-15.md
|
||||
|
||||
# Shows line-by-line attribution:
|
||||
a1b2c3d4 (Rep. Nancy Pelosi 2021-03-11) (a) In general.—The Secretary, acting through
|
||||
e5f6g7h8 (Sen. Chuck Schumer 2021-03-11) the Director of the Centers for Disease Control
|
||||
f9g0h1i2 (Rep. Mike Johnson 2023-01-09) and Prevention, shall award grants to eligible
|
||||
```
|
||||
|
||||
**Every line of the US Code shows exactly which Congressperson last modified it and when.**
|
||||
|
||||
## The Vision
|
||||
|
||||
This system transforms US Code tracking from annual snapshots to **line-level legislative history**:
|
||||
|
||||
- **📍 Granular Attribution**: Every line shows the exact Congressperson who last changed it
|
||||
- **🕰️ Complete Timeline**: Full evolution from 2013 to present with chronological commits
|
||||
- **📊 Rich Context**: Committee reports, debates, sponsor details, and legislative process
|
||||
- **🔍 Powerful Queries**: `git log --follow Section-280g-15.md` to see complete section history
|
||||
- **🎯 Diff Analysis**: `git diff PL-116-260..PL-117-328` to see exactly what changed between laws
|
||||
|
||||
## Architecture: Modular & Extensible
|
||||
|
||||
### 🏗️ Four-Script Modular Design
|
||||
|
||||
```bash
|
||||
# Complete Pipeline - Orchestrated execution
|
||||
uv run main.py # Run all stages with defaults
|
||||
uv run main.py --comprehensive # Full download with all data sources
|
||||
uv run main.py --force-migration # Force re-migration of existing files
|
||||
|
||||
# Individual Stages - Independent execution
|
||||
uv run main.py --stage 1 # Download & cache data only
|
||||
uv run main.py --stage 2 # Migrate cached data to JSON
|
||||
uv run main.py --stage 3 # Generate git commit plans
|
||||
uv run main.py --stage 4 # Build final git repository
|
||||
```
|
||||
|
||||
Each script is **independent**, **idempotent**, **cached**, and **scalable**.
|
||||
|
||||
### 📊 Comprehensive Data Sources
|
||||
|
||||
Sources:
|
||||
|
||||
- https://www.govinfo.gov/bulkdata/
|
||||
- https://xml.house.gov/
|
||||
- https://uscode.house.gov/download/priorreleasepoints.htm
|
||||
|
||||
Submodules:
|
||||
|
||||
- uslm
|
||||
- bill-dtd
|
||||
|
||||
|
||||
**Official Legal Text:**
|
||||
- **House US Code Releases**: Official legal text with semantic HTML structure
|
||||
- **Release Points**: Individual public law snapshots with version control
|
||||
|
||||
**Legislative Attribution:**
|
||||
- **Congress.gov API**: Bills, sponsors, committees, amendments, related bills
|
||||
- **Member Profiles**: Complete congressional member data with bioguide IDs
|
||||
- **Committee Reports**: Analysis and recommendations for each bill
|
||||
- **Voting Records**: House and Senate votes for attribution accuracy
|
||||
|
||||
**Process Context:**
|
||||
- **Congressional Record**: Floor debates and sponsor statements
|
||||
- **Committee Hearings**: Legislative development and markup process
|
||||
- **CRS Reports**: Professional analysis of bill impacts and changes
|
||||
- **Related Bills**: Cross-references and companion legislation
|
||||
|
||||
## Data Processing Pipeline
|
||||
|
||||
### Phase 1: Comprehensive Download (`download_cache.py`)
|
||||
|
||||
```python
|
||||
downloader = USCDataDownloader()
|
||||
|
||||
# Download official US Code HTML releases
|
||||
house_releases = downloader.download_house_usc_releases(public_laws)
|
||||
|
||||
# Fetch comprehensive bill data from Congress.gov API
|
||||
bill_data = downloader.download_congress_api_bills(public_laws)
|
||||
|
||||
# Get member profiles for proper attribution
|
||||
members = downloader.download_member_profiles(congresses=[113,114,115,116,117,118,119])
|
||||
|
||||
# Download committee reports and analysis
|
||||
committee_data = downloader.download_committee_reports(public_laws)
|
||||
```
|
||||
|
||||
**Features:**
|
||||
- ✅ **Smart Caching**: Never re-download existing data - fully idempotent
|
||||
- ✅ **Rate Limiting**: Respects Congress.gov 1,000 req/hour limit
|
||||
- ✅ **Rich Metadata**: Tracks download timestamps, sizes, sources
|
||||
- ✅ **Error Recovery**: Continues processing despite individual failures
|
||||
- ✅ **Organized Storage**: Separate cache directories by data type
|
||||
- ✅ **Cache Validation**: `is_cached()` checks prevent duplicate downloads
|
||||
|
||||
### Phase 2: Data Normalization (`migrate_to_datastore.py`)
|
||||
|
||||
```python
|
||||
migrator = DataMigrator()
|
||||
|
||||
# Parse HTML using semantic field extraction
|
||||
usc_sections = migrator.extract_usc_sections_from_html(house_releases)
|
||||
|
||||
# Normalize congressional data with Pydantic validation
|
||||
normalized_bills = migrator.migrate_congress_api_data(bill_data)
|
||||
|
||||
# Cross-reference and validate all relationships
|
||||
migrator.validate_and_index(usc_sections, normalized_bills, members)
|
||||
```
|
||||
|
||||
**Features:**
|
||||
- ✅ **HTML Parsing**: Extract clean USC text from semantic HTML fields
|
||||
- ✅ **Structure Normalization**: Handle multiple conversion program versions
|
||||
- ✅ **Pydantic Validation**: Type safety and business rule enforcement
|
||||
- ✅ **Cross-Referencing**: Link bills to public laws to USC changes
|
||||
- ✅ **Data Integrity**: Comprehensive validation and consistency checks
|
||||
- ✅ **Idempotent Processing**: Skip existing output files, `--force-migration` to override
|
||||
- ✅ **Output Validation**: Checks for existing `data/usc_sections/{law}.json` files
|
||||
|
||||
### Phase 3: Smart Git Planning (`generate_git_plan.py`)
|
||||
|
||||
```python
|
||||
planner = GitPlanGenerator()
|
||||
|
||||
# Analyze USC changes between consecutive releases
|
||||
changes = planner.analyze_usc_changes(old_release, new_release)
|
||||
|
||||
# Generate commit plans for each public law
|
||||
commit_plans = planner.generate_incremental_commit_plans(changes, public_laws)
|
||||
|
||||
# Optimize commit sequence for git blame accuracy
|
||||
optimized = planner.optimize_commit_sequence(commit_plans)
|
||||
```
|
||||
|
||||
**Features:**
|
||||
- ✅ **Section-Level Diff**: Track changes at USC section granularity
|
||||
- ✅ **Incremental Commits**: Only commit files that actually changed
|
||||
- ✅ **Smart Attribution**: Map changes to specific public laws and sponsors
|
||||
- ✅ **Chronological Order**: Proper timestamp ordering for git history
|
||||
- ✅ **Conflict Resolution**: Handle complex multi-law interactions
|
||||
- ✅ **Plan Caching**: Saves commit plans to `data/git_plans/` for reuse
|
||||
- ✅ **Input Validation**: Checks for required USC sections data before planning
|
||||
|
||||
### Phase 4: Repository Construction (`build_git_repo.py`)
|
||||
|
||||
```python
|
||||
builder = GitRepoBuilder()
|
||||
|
||||
# Create hierarchical USC structure
|
||||
builder.build_hierarchical_structure(usc_sections)
|
||||
|
||||
# Apply commit plans with proper attribution
|
||||
for plan in commit_plans:
|
||||
builder.apply_commit_plan(plan)
|
||||
|
||||
# Validate git blame functionality
|
||||
builder.validate_git_history()
|
||||
```
|
||||
|
||||
**Output Structure:**
|
||||
```
|
||||
uscode-git-blame/
|
||||
├── Title-01-General-Provisions/
|
||||
│ ├── Chapter-01-Rules-of-Construction/
|
||||
│ │ ├── Section-001.md # § 1. Words denoting number, gender...
|
||||
│ │ ├── Section-002.md # § 2. "County" as including "parish"...
|
||||
│ │ └── Section-008.md # § 8. "Person", "human being"...
|
||||
│ └── Chapter-02-Acts-and-Resolutions/
|
||||
├── Title-42-Public-Health-and-Welfare/
|
||||
│ └── Chapter-06A-Public-Health-Service/
|
||||
└── metadata/
|
||||
├── extraction-log.json
|
||||
├── commit-plans.json
|
||||
└── validation-results.json
|
||||
```
|
||||
|
||||
**Features:**
|
||||
- ✅ **Hierarchical Organization**: Title/Chapter/Section file structure
|
||||
- ✅ **Clean Markdown**: Convert HTML to readable markdown with proper formatting
|
||||
- ✅ **Proper Attribution**: Git author/committer fields with congressional sponsors
|
||||
- ✅ **Rich Commit Messages**: Include bill details, affected sections, sponsor quotes
|
||||
- ✅ **Git Blame Validation**: Verify every line has proper attribution
|
||||
- ✅ **Repository Management**: `--force-rebuild` flag for clean repository recreation
|
||||
- ✅ **Build Metadata**: Comprehensive statistics in `metadata/` directory
|
||||
|
||||
## Advanced Features
|
||||
|
||||
### ⚡ Idempotent & Cached Processing
|
||||
|
||||
**All scripts implement comprehensive caching and idempotency:**
|
||||
|
||||
```bash
|
||||
# First run - downloads and processes everything
|
||||
uv run main.py --laws 119-001,119-004
|
||||
|
||||
# Second run - skips existing work, completes instantly
|
||||
uv run main.py --laws 119-001,119-004
|
||||
# Output: ✅ Skipping HTML migration for 119-001 - output exists
|
||||
|
||||
# Force complete re-processing when needed
|
||||
uv run main.py --laws 119-001,119-004 --force-migration --force-rebuild
|
||||
```
|
||||
|
||||
**Script-Level Caching:**
|
||||
- **Stage 1**: `download_cache/` - Never re-download existing files
|
||||
- **Stage 2**: `data/usc_sections/` - Skip processing if JSON output exists
|
||||
- **Stage 3**: `data/git_plans/` - Reuse existing commit plans
|
||||
- **Stage 4**: Repository exists check with `--force-rebuild` override
|
||||
|
||||
**Benefits:**
|
||||
- ✅ **Development Speed**: Instant re-runs during development
|
||||
- ✅ **Production Safety**: Resume interrupted processes seamlessly
|
||||
- ✅ **Resource Efficiency**: No redundant API calls or processing
|
||||
- ✅ **Incremental Updates**: Process only new public laws
|
||||
- ✅ **Debugging Support**: Test individual stages without full pipeline
|
||||
|
||||
### 🔍 Intelligent Text Extraction
|
||||
|
||||
**Multi-Version HTML Parsing:**
|
||||
- Handles House conversion programs: `xy2html.pm-0.400` through `xy2html.pm-0.401`
|
||||
- Extracts clean text from semantic field markers (`<!-- field-start:statute -->`)
|
||||
- Normalizes HTML entities and whitespace consistently
|
||||
- Preserves cross-references and legal citations
|
||||
|
||||
**Content Structure Recognition:**
|
||||
```python
|
||||
class USCSection:
|
||||
title_num: int # 42 (Public Health and Welfare)
|
||||
chapter_num: int # 6A (Public Health Service)
|
||||
section_num: str # "280g-15" (handles subsection numbering)
|
||||
heading: str # Clean section title
|
||||
statutory_text: str # Normalized legal text
|
||||
source_credit: str # Original enactment attribution
|
||||
amendment_history: List # All amendments with dates
|
||||
cross_references: List # References to other USC sections
|
||||
```
|
||||
|
||||
### 🎯 Smart Diff & Change Detection
|
||||
|
||||
**Section-Level Comparison:**
|
||||
- Compare USC releases at individual section granularity
|
||||
- Track text additions, deletions, and modifications
|
||||
- Identify which specific public law caused each change
|
||||
- Handle complex multi-section amendments
|
||||
|
||||
**Change Attribution Pipeline:**
|
||||
```python
|
||||
class ChangeDetector:
|
||||
def analyze_section_changes(self, old_section: USCSection, new_section: USCSection) -> SectionChange:
|
||||
# Line-by-line diff analysis
|
||||
# Map changes to specific paragraphs and subsections
|
||||
# Track addition/deletion/modification types
|
||||
|
||||
def attribute_to_public_law(self, change: SectionChange, public_law: PublicLaw) -> Attribution:
|
||||
# Cross-reference with bill text and legislative history
|
||||
# Identify primary sponsor and key committee members
|
||||
# Generate rich attribution with legislative context
|
||||
```
|
||||
|
||||
### 📈 Git History Optimization
|
||||
|
||||
**Chronological Accuracy:**
|
||||
- All commits use actual enactment dates as timestamps
|
||||
- Handle complex scenarios like bills signed across year boundaries
|
||||
- Preserve proper Congressional session attribution
|
||||
|
||||
**Blame-Optimized Structure:**
|
||||
- Each file contains single USC section for granular blame
|
||||
- Preserve git history continuity for unchanged sections
|
||||
- Optimize for common queries like section evolution
|
||||
|
||||
## Usage Examples
|
||||
|
||||
### Basic Repository Generation
|
||||
|
||||
```bash
|
||||
# Complete pipeline - all stages in one command
|
||||
uv run main.py
|
||||
|
||||
# Comprehensive processing with all data sources
|
||||
uv run main.py --comprehensive
|
||||
|
||||
# Process specific public laws
|
||||
uv run main.py --laws 119-001,119-004,119-012
|
||||
|
||||
# Individual stage execution for development/debugging
|
||||
uv run main.py --stage 1 # Download only
|
||||
uv run main.py --stage 2 # Migration only
|
||||
uv run main.py --stage 3 # Planning only
|
||||
uv run main.py --stage 4 # Repository building only
|
||||
```
|
||||
|
||||
### Advanced Queries
|
||||
|
||||
```bash
|
||||
cd uscode-git-blame
|
||||
|
||||
# See who last modified healthcare provisions
|
||||
git blame Title-42-Public-Health-and-Welfare/Chapter-06A-Public-Health-Service/Section-280g-15.md
|
||||
|
||||
# Track complete evolution of a section
|
||||
git log --follow --patch Title-42-Public-Health-and-Welfare/Chapter-06A-Public-Health-Service/Section-280g-15.md
|
||||
|
||||
# Compare major healthcare laws
|
||||
git diff PL-111-148..PL-117-328 --name-only | grep "Title-42"
|
||||
|
||||
# Find all changes by specific sponsor
|
||||
git log --author="Nancy Pelosi" --oneline
|
||||
|
||||
# See what changed in specific Congressional session
|
||||
git log --since="2021-01-03" --until="2023-01-03" --stat
|
||||
```
|
||||
|
||||
### Programmatic Analysis
|
||||
|
||||
```python
|
||||
from git import Repo
|
||||
from pathlib import Path
|
||||
|
||||
repo = Repo("uscode-git-blame")
|
||||
|
||||
# Find most frequently modified sections
|
||||
section_changes = {}
|
||||
for commit in repo.iter_commits():
|
||||
for file in commit.stats.files:
|
||||
section_changes[file] = section_changes.get(file, 0) + 1
|
||||
|
||||
# Analyze sponsor activity
|
||||
sponsor_activity = {}
|
||||
for commit in repo.iter_commits():
|
||||
author = commit.author.name
|
||||
sponsor_activity[author] = sponsor_activity.get(author, 0) + 1
|
||||
|
||||
# Track healthcare law evolution
|
||||
healthcare_commits = [c for c in repo.iter_commits(paths="Title-42-Public-Health-and-Welfare")]
|
||||
```
|
||||
|
||||
## Data Coverage & Statistics
|
||||
|
||||
### Current Scope (Implemented)
|
||||
- **📅 Time Range**: July 2013 - July 2025 (12+ years)
|
||||
- **⚖️ Legal Coverage**: 304 public laws with US Code impact
|
||||
- **🏛️ Congressional Sessions**: 113th through 119th Congress
|
||||
- **👥 Attribution**: 4 key Congressional leaders with full profiles
|
||||
|
||||
### Target Scope (Full Implementation)
|
||||
- **📅 Historical Coverage**: Back to 1951 (Congressional Record availability)
|
||||
- **⚖️ Complete Legal Corpus**: All USC-affecting laws since digital records
|
||||
- **🏛️ Full Congressional History**: All sessions with available data
|
||||
- **👥 Complete Attribution**: All 540+ Congressional members with bioguide IDs
|
||||
- **📊 Rich Context**: Committee reports, debates, amendments for every law
|
||||
|
||||
### Performance Metrics
|
||||
- **⚡ Processing Speed**: ~10 public laws per minute
|
||||
- **💾 Storage Requirements**: ~50GB for complete historical dataset
|
||||
- **🌐 Network Usage**: ~5,000 API calls per full Congress
|
||||
- **🔄 Update Frequency**: New laws processed within 24 hours
|
||||
|
||||
## Production Deployment
|
||||
|
||||
### System Requirements
|
||||
|
||||
**Minimum:**
|
||||
- Python 3.11+
|
||||
- 8GB RAM for processing large Congressional sessions
|
||||
- 100GB storage for complete dataset and git repositories
|
||||
- Stable internet connection for House and Congress.gov APIs
|
||||
|
||||
**Recommended:**
|
||||
- Python 3.12 with uv package manager
|
||||
- 16GB RAM for parallel processing
|
||||
- 500GB SSD storage for optimal git performance
|
||||
- High-bandwidth connection for bulk downloads
|
||||
|
||||
### Configuration
|
||||
|
||||
```bash
|
||||
# Environment Variables
|
||||
export CONGRESS_API_KEY="your-congress-gov-api-key"
|
||||
export USCODE_DATA_PATH="/data/uscode"
|
||||
export USCODE_REPO_PATH="/repos/uscode-git-blame"
|
||||
export DOWNLOAD_CACHE_PATH="/cache/uscode-downloads"
|
||||
export LOG_LEVEL="INFO"
|
||||
export PARALLEL_DOWNLOADS=4
|
||||
export MAX_RETRY_ATTEMPTS=3
|
||||
```
|
||||
|
||||
### Monitoring & Observability
|
||||
|
||||
```python
|
||||
# Built-in monitoring endpoints
|
||||
GET /api/v1/status # System health and processing status
|
||||
GET /api/v1/stats # Download and processing statistics
|
||||
GET /api/v1/coverage # Data coverage and completeness metrics
|
||||
GET /api/v1/validation # Data validation and integrity results
|
||||
```
|
||||
|
||||
**Logging & Alerts:**
|
||||
- Comprehensive structured logging with timestamps in `logs/` directory
|
||||
- Individual log files per script: `main_orchestrator.log`, `download_cache.log`, etc.
|
||||
- Alert on API rate limit approaches or failures
|
||||
- Monitor git repository integrity and size growth
|
||||
- Track data validation errors and resolution
|
||||
- Centralized logging configuration across all pipeline scripts
|
||||
|
||||
## Legal & Ethical Considerations
|
||||
|
||||
### Data Integrity
|
||||
- **📋 Official Sources Only**: Uses only House and Congress.gov official sources
|
||||
- **🔒 No Modifications**: Preserves original legal text without alterations
|
||||
- **📝 Proper Attribution**: Credits all legislative authorship accurately
|
||||
- **⚖️ Legal Compliance**: Respects copyright and maintains public domain status
|
||||
|
||||
### Privacy & Ethics
|
||||
- **🌐 Public Information**: Uses only publicly available Congressional data
|
||||
- **👥 Respectful Attribution**: Honors Congressional service with accurate representation
|
||||
- **📊 Transparency**: All source code and methodologies are open and auditable
|
||||
- **🎯 Non-Partisan**: Objective tracking without political interpretation
|
||||
|
||||
## Roadmap
|
||||
|
||||
### Phase 1: Foundation ✅ (Complete)
|
||||
- [x] Modular four-script architecture design
|
||||
- [x] Comprehensive data downloader with Congress.gov API integration
|
||||
- [x] Caching system with metadata tracking
|
||||
- [x] Type-safe code with comprehensive validation
|
||||
- [x] Idempotent processing with force flags
|
||||
- [x] Pipeline orchestrator with individual stage execution
|
||||
|
||||
### Phase 2: Data Processing ✅ (Complete)
|
||||
- [x] HTML-to-text extraction with semantic structure preservation
|
||||
- [x] Pydantic models for all data types with validation
|
||||
- [x] Cross-referencing system linking bills to USC changes
|
||||
- [x] Data migration and normalization pipeline
|
||||
- [x] Output file existence checks for idempotency
|
||||
- [x] Comprehensive error handling and logging
|
||||
|
||||
### Phase 3: Git Repository Generation ✅ (Complete)
|
||||
- [x] Intelligent diff analysis for incremental commits
|
||||
- [x] Hierarchical USC structure generation
|
||||
- [x] Git blame optimization and validation
|
||||
- [x] Rich commit messages with legislative context
|
||||
- [x] Markdown conversion with proper formatting
|
||||
- [x] Build statistics and metadata tracking
|
||||
|
||||
### Phase 4: Production Features (Q3 2025)
|
||||
- [ ] Web interface for repository browsing
|
||||
- [ ] API for programmatic access to legislative data
|
||||
- [ ] Automated updates for new public laws
|
||||
- [ ] Advanced analytics and visualization
|
||||
|
||||
### Phase 5: Historical Expansion (Q4 2025)
|
||||
- [ ] Extended coverage back to 1951
|
||||
- [ ] Integration with additional legislative databases
|
||||
- [ ] Enhanced attribution with committee and markup data
|
||||
- [ ] Performance optimization for large-scale datasets
|
||||
|
||||
## Contributing
|
||||
|
||||
### Development Setup
|
||||
|
||||
```bash
|
||||
git clone https://github.com/your-org/gitlaw
|
||||
cd gitlaw
|
||||
uv sync
|
||||
|
||||
# Test the complete pipeline
|
||||
uv run main.py --help
|
||||
|
||||
# Run individual stages for development
|
||||
uv run main.py --stage 1 --laws 119-001 # Test download
|
||||
uv run main.py --stage 2 --laws 119-001 # Test migration
|
||||
uv run main.py --stage 3 --laws 119-001 # Test planning
|
||||
uv run main.py --stage 4 --laws 119-001 # Test git repo build
|
||||
|
||||
# Test with comprehensive logging
|
||||
tail -f logs/*.log # Monitor all pipeline logs
|
||||
```
|
||||
|
||||
### Adding New Features
|
||||
|
||||
1. **Data Sources**: Extend `download_cache.py` with new Congress.gov endpoints
|
||||
2. **Processing**: Add new Pydantic models in `models.py`
|
||||
3. **Git Features**: Enhance `build_git_repo.py` with new attribution methods
|
||||
4. **Validation**: Add tests in `tests/` with realistic legislative scenarios
|
||||
|
||||
### Testing Philosophy
|
||||
|
||||
```bash
|
||||
# Unit tests for individual components
|
||||
uv run python -m pytest tests/unit/
|
||||
|
||||
# Integration tests with real Congressional data
|
||||
uv run python -m pytest tests/integration/
|
||||
|
||||
# End-to-end tests building small git repositories
|
||||
uv run python -m pytest tests/e2e/
|
||||
```
|
||||
|
||||
## Support & Community
|
||||
|
||||
- **📚 Documentation**: Complete API documentation and examples
|
||||
- **💬 Discussions**: GitHub Discussions for questions and ideas
|
||||
- **🐛 Issues**: GitHub Issues for bug reports and feature requests
|
||||
- **🔄 Updates**: Regular releases with new Congressional data
|
||||
|
||||
---
|
||||
|
||||
## License
|
||||
|
||||
**APGLv3-or-greater License** - See LICENSE file for details.
|
||||
|
||||
*The United States Code is in the public domain. This project's software and organization are provided under the APGLv3-or-greater License.*
|
||||
|
||||
---
|
||||
|
||||
**🏛️ "Every line of law, attributed to its author, tracked through time."**
|
||||
|
||||
*Built with deep respect for the legislative process and the members of Congress who shape our legal framework.*
|
||||
719
build_git_repo.py
Normal file
719
build_git_repo.py
Normal file
@@ -0,0 +1,719 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
USC Git Blame Repository Builder
|
||||
|
||||
Executes git commit plans to build the final blame-enabled repository:
|
||||
|
||||
1. Creates hierarchical USC file structure (Title/Chapter/Section)
|
||||
2. Converts HTML to clean markdown with proper formatting
|
||||
3. Executes git commits with proper attribution and timestamps
|
||||
4. Validates git blame functionality and attribution accuracy
|
||||
5. Generates repository metadata and documentation
|
||||
|
||||
Architecture: Download → Cache → Migrate → Plan → **Build**
|
||||
This script handles the final step: git repository construction.
|
||||
"""
|
||||
|
||||
import os
|
||||
import json
|
||||
import subprocess
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
from typing import Dict, List, Optional, Any
|
||||
import logging
|
||||
import html
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
|
||||
# Configure logging
|
||||
logs_dir = Path('logs')
|
||||
logs_dir.mkdir(exist_ok=True)
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
|
||||
handlers=[
|
||||
logging.FileHandler(logs_dir / 'build_git_repo.log'),
|
||||
logging.StreamHandler()
|
||||
]
|
||||
)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class BuildStatistics:
|
||||
"""Statistics for repository build process"""
|
||||
commits_executed: int = 0
|
||||
files_created: int = 0
|
||||
files_modified: int = 0
|
||||
files_deleted: int = 0
|
||||
total_lines_added: int = 0
|
||||
total_lines_deleted: int = 0
|
||||
build_duration_seconds: float = 0.0
|
||||
git_repo_size_mb: float = 0.0
|
||||
validation_passed: bool = False
|
||||
|
||||
@property
|
||||
def total_file_operations(self) -> int:
|
||||
return self.files_created + self.files_modified + self.files_deleted
|
||||
|
||||
|
||||
class MarkdownConverter:
|
||||
"""Converts USC HTML content to clean markdown format"""
|
||||
|
||||
def __init__(self):
|
||||
self.html_entities = {
|
||||
'—': '—',
|
||||
'“': '"',
|
||||
'”': '"',
|
||||
'‘': ''',
|
||||
'’': ''',
|
||||
' ': ' ',
|
||||
'&': '&',
|
||||
'<': '<',
|
||||
'>': '>',
|
||||
'§': '§'
|
||||
}
|
||||
|
||||
def convert_section_to_markdown(self, section_data: Dict[str, Any]) -> str:
|
||||
"""Convert USC section data to formatted markdown"""
|
||||
|
||||
lines = []
|
||||
|
||||
# Section header
|
||||
section_id = section_data.get('section_id', 'Unknown')
|
||||
heading = section_data.get('heading', '')
|
||||
section_num = section_data.get('section_num', '')
|
||||
|
||||
lines.append(f"# § {section_num}. {heading}")
|
||||
lines.append("")
|
||||
|
||||
# Main statutory text
|
||||
statutory_text = section_data.get('statutory_text', '')
|
||||
if statutory_text:
|
||||
clean_text = self._clean_and_format_text(statutory_text)
|
||||
lines.append(clean_text)
|
||||
lines.append("")
|
||||
|
||||
# Source credit
|
||||
source_credit = section_data.get('source_credit', '')
|
||||
if source_credit:
|
||||
lines.append("## Source")
|
||||
lines.append("")
|
||||
lines.append(self._clean_text(source_credit))
|
||||
lines.append("")
|
||||
|
||||
# Amendment history
|
||||
amendment_history = section_data.get('amendment_history', [])
|
||||
if amendment_history:
|
||||
lines.append("## Amendment History")
|
||||
lines.append("")
|
||||
for amendment in amendment_history:
|
||||
clean_amendment = self._clean_text(amendment)
|
||||
if clean_amendment.strip():
|
||||
lines.append(f"- {clean_amendment}")
|
||||
lines.append("")
|
||||
|
||||
# Metadata
|
||||
lines.append("---")
|
||||
lines.append("")
|
||||
lines.append("**USC Section Metadata:**")
|
||||
lines.append(f"- Section ID: `{section_id}`")
|
||||
lines.append(f"- Title: {section_data.get('title_num', 'Unknown')}")
|
||||
lines.append(f"- Chapter: {section_data.get('chapter_num', 'Unknown')}")
|
||||
lines.append(f"- Enacted Through: {section_data.get('enacted_through', 'Unknown')}")
|
||||
lines.append("")
|
||||
lines.append("*Generated by USC Git Blame System*")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
def _clean_and_format_text(self, text: str) -> str:
|
||||
"""Clean and format statutory text with proper paragraphs"""
|
||||
|
||||
# Clean HTML entities
|
||||
clean_text = self._clean_text(text)
|
||||
|
||||
# Split into paragraphs and format
|
||||
paragraphs = []
|
||||
current_paragraph = []
|
||||
|
||||
for line in clean_text.split('\n'):
|
||||
line = line.strip()
|
||||
if not line:
|
||||
if current_paragraph:
|
||||
paragraphs.append(' '.join(current_paragraph))
|
||||
current_paragraph = []
|
||||
else:
|
||||
current_paragraph.append(line)
|
||||
|
||||
if current_paragraph:
|
||||
paragraphs.append(' '.join(current_paragraph))
|
||||
|
||||
# Format paragraphs with proper indentation for subsections
|
||||
formatted_paragraphs = []
|
||||
for para in paragraphs:
|
||||
# Detect subsection patterns like "(a)", "(1)", etc.
|
||||
if re.match(r'^\([a-zA-Z0-9]+\)', para.strip()):
|
||||
formatted_paragraphs.append(f"**{para}**")
|
||||
else:
|
||||
formatted_paragraphs.append(para)
|
||||
|
||||
return '\n\n'.join(formatted_paragraphs)
|
||||
|
||||
def _clean_text(self, text: str) -> str:
|
||||
"""Clean HTML entities and normalize whitespace"""
|
||||
|
||||
# Decode HTML entities
|
||||
clean = html.unescape(text)
|
||||
|
||||
# Replace specific entities
|
||||
for entity, replacement in self.html_entities.items():
|
||||
clean = clean.replace(entity, replacement)
|
||||
|
||||
# Normalize whitespace
|
||||
clean = re.sub(r'\s+', ' ', clean)
|
||||
clean = clean.strip()
|
||||
|
||||
return clean
|
||||
|
||||
|
||||
class GitRepositoryBuilder:
|
||||
"""Builds the final USC git repository from commit plans"""
|
||||
|
||||
def __init__(self, repo_path: Path = Path("uscode-git-blame")):
|
||||
self.repo_path = repo_path
|
||||
self.markdown_converter = MarkdownConverter()
|
||||
self.stats = BuildStatistics()
|
||||
self.build_start_time = datetime.now()
|
||||
|
||||
# Ensure git is available
|
||||
self._check_git_availability()
|
||||
|
||||
def _check_git_availability(self):
|
||||
"""Verify git is installed and available"""
|
||||
try:
|
||||
subprocess.run(['git', '--version'], check=True, capture_output=True)
|
||||
logger.info("✅ Git is available")
|
||||
except (subprocess.CalledProcessError, FileNotFoundError):
|
||||
raise RuntimeError("Git is not installed or not available in PATH")
|
||||
|
||||
def initialize_repository(self, force: bool = False) -> None:
|
||||
"""Initialize a new git repository"""
|
||||
|
||||
if self.repo_path.exists() and force:
|
||||
logger.warning(f"🗑️ Removing existing repository: {self.repo_path}")
|
||||
shutil.rmtree(self.repo_path)
|
||||
elif self.repo_path.exists():
|
||||
raise ValueError(f"Repository already exists: {self.repo_path}. Use force=True to overwrite.")
|
||||
|
||||
# Create repository directory
|
||||
self.repo_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Initialize git repository
|
||||
self._run_git_command(['init'], "Initialize git repository")
|
||||
|
||||
# Configure git for USC commits
|
||||
self._run_git_command(['config', 'user.name', 'USC Git Blame System'], "Set git user name")
|
||||
self._run_git_command(['config', 'user.email', 'system@uscode.git'], "Set git user email")
|
||||
|
||||
# Create initial directory structure
|
||||
self._create_directory_structure()
|
||||
|
||||
logger.info(f"✅ Repository initialized: {self.repo_path}")
|
||||
|
||||
def _create_directory_structure(self) -> None:
|
||||
"""Create the hierarchical USC directory structure"""
|
||||
|
||||
# Create metadata directory
|
||||
metadata_dir = self.repo_path / "metadata"
|
||||
metadata_dir.mkdir(exist_ok=True)
|
||||
|
||||
# Create initial README
|
||||
readme_content = """# United States Code - Git Blame Repository
|
||||
|
||||
This repository contains the complete United States Code with line-by-line attribution
|
||||
to Congressional sponsors using git blame functionality.
|
||||
|
||||
## Structure
|
||||
|
||||
```
|
||||
Title-XX-Title-Name/
|
||||
├── Chapter-YY-Chapter-Name/
|
||||
│ ├── Section-ZZZZ.md
|
||||
│ └── Section-AAAA.md
|
||||
└── metadata/
|
||||
├── extraction-log.json
|
||||
└── build-statistics.json
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
```bash
|
||||
# See who last modified a specific section
|
||||
git blame Title-42-Public-Health-and-Welfare/Chapter-06A-Public-Health-Service/Section-280g-15.md
|
||||
|
||||
# Track complete evolution of a section
|
||||
git log --follow --patch Title-42-Public-Health-and-Welfare/Chapter-06A-Public-Health-Service/Section-280g-15.md
|
||||
|
||||
# Find all changes by a specific sponsor
|
||||
git log --author="Nancy Pelosi" --oneline
|
||||
```
|
||||
|
||||
## Data Sources
|
||||
|
||||
- **Legal Text**: House Office of Law Revision Counsel
|
||||
- **Attribution**: Congress.gov API
|
||||
- **Generated**: USC Git Blame System
|
||||
|
||||
---
|
||||
|
||||
*Every line shows exactly which Congressperson last modified it and when.*
|
||||
"""
|
||||
|
||||
readme_path = self.repo_path / "README.md"
|
||||
readme_path.write_text(readme_content)
|
||||
|
||||
logger.info("📁 Directory structure created")
|
||||
|
||||
def execute_commit_plans(self, plans_file: Path) -> None:
|
||||
"""Execute all commit plans to build the repository"""
|
||||
|
||||
logger.info(f"🚀 Executing commit plans from {plans_file}")
|
||||
|
||||
# Load commit plans
|
||||
with open(plans_file, 'r') as f:
|
||||
plans_data = json.load(f)
|
||||
|
||||
commits = plans_data.get('commits', [])
|
||||
metadata = plans_data.get('metadata', {})
|
||||
|
||||
logger.info(f"📋 Found {len(commits)} commits to execute")
|
||||
logger.info(f"📊 Plans generated: {metadata.get('generated_at', 'Unknown')}")
|
||||
|
||||
# Execute each commit in order
|
||||
for i, commit_data in enumerate(commits):
|
||||
logger.info(f"🔄 Executing commit {i+1}/{len(commits)}: {commit_data['public_law_id']}")
|
||||
|
||||
success = self._execute_single_commit(commit_data)
|
||||
if success:
|
||||
self.stats.commits_executed += 1
|
||||
else:
|
||||
logger.error(f"❌ Failed to execute commit for {commit_data['public_law_id']}")
|
||||
|
||||
# Create final metadata
|
||||
self._generate_repository_metadata(metadata)
|
||||
|
||||
logger.info(f"✅ Repository build complete: {self.stats.commits_executed}/{len(commits)} commits executed")
|
||||
|
||||
def _execute_single_commit(self, commit_data: Dict[str, Any]) -> bool:
|
||||
"""Execute a single git commit from the plan"""
|
||||
|
||||
try:
|
||||
public_law_id = commit_data['public_law_id']
|
||||
|
||||
# Apply file changes
|
||||
files_changed = commit_data.get('files_changed', [])
|
||||
for file_change in files_changed:
|
||||
success = self._apply_file_change(file_change, public_law_id)
|
||||
if not success:
|
||||
logger.warning(f"⚠️ Failed to apply file change: {file_change.get('file_path')}")
|
||||
|
||||
# Stage all changes
|
||||
self._run_git_command(['add', '.'], f"Stage changes for {public_law_id}")
|
||||
|
||||
# Check if there are actually changes to commit
|
||||
result = subprocess.run(['git', 'diff', '--cached', '--name-only'],
|
||||
cwd=self.repo_path, capture_output=True, text=True)
|
||||
|
||||
if not result.stdout.strip():
|
||||
logger.warning(f"⚠️ No changes to commit for {public_law_id}")
|
||||
return False
|
||||
|
||||
# Create commit with proper attribution and timestamp
|
||||
commit_message = commit_data['message']['title']
|
||||
commit_body = commit_data['message']['body']
|
||||
full_message = f"{commit_message}\n\n{commit_body}"
|
||||
|
||||
# Set author and committer info
|
||||
author = commit_data['author']
|
||||
commit_date = commit_data['commit_date']
|
||||
|
||||
env = os.environ.copy()
|
||||
env.update({
|
||||
'GIT_AUTHOR_NAME': author['name'],
|
||||
'GIT_AUTHOR_EMAIL': author['email'],
|
||||
'GIT_AUTHOR_DATE': commit_date,
|
||||
'GIT_COMMITTER_NAME': author['name'],
|
||||
'GIT_COMMITTER_EMAIL': author['email'],
|
||||
'GIT_COMMITTER_DATE': commit_date
|
||||
})
|
||||
|
||||
# Create commit
|
||||
subprocess.run(['git', 'commit', '-m', full_message],
|
||||
cwd=self.repo_path, check=True, env=env)
|
||||
|
||||
# Apply tags if specified
|
||||
tags = commit_data.get('metadata', {}).get('tags', [])
|
||||
for tag in tags:
|
||||
try:
|
||||
subprocess.run(['git', 'tag', tag],
|
||||
cwd=self.repo_path, check=True)
|
||||
except subprocess.CalledProcessError:
|
||||
logger.warning(f"⚠️ Failed to create tag: {tag}")
|
||||
|
||||
logger.debug(f"✅ Committed {public_law_id}: {len(files_changed)} files")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"❌ Error executing commit for {commit_data.get('public_law_id')}: {e}")
|
||||
return False
|
||||
|
||||
def _apply_file_change(self, file_change: Dict[str, Any], public_law_id: str) -> bool:
|
||||
"""Apply a single file change (add, modify, or delete)"""
|
||||
|
||||
try:
|
||||
file_path = file_change['file_path']
|
||||
change_type = file_change['change_type']
|
||||
section_id = file_change['section_id']
|
||||
|
||||
full_path = self.repo_path / file_path
|
||||
|
||||
if change_type == "deleted":
|
||||
if full_path.exists():
|
||||
full_path.unlink()
|
||||
self.stats.files_deleted += 1
|
||||
logger.debug(f"🗑️ Deleted: {file_path}")
|
||||
return True
|
||||
|
||||
elif change_type in ["added", "modified"]:
|
||||
# Load section data to generate content
|
||||
section_data = self._load_section_data(section_id, public_law_id)
|
||||
if not section_data:
|
||||
logger.warning(f"⚠️ No section data found for {section_id}")
|
||||
return False
|
||||
|
||||
# Create parent directories
|
||||
full_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Convert to markdown
|
||||
markdown_content = self.markdown_converter.convert_section_to_markdown(section_data)
|
||||
|
||||
# Write file
|
||||
full_path.write_text(markdown_content, encoding='utf-8')
|
||||
|
||||
if change_type == "added":
|
||||
self.stats.files_created += 1
|
||||
logger.debug(f"➕ Added: {file_path}")
|
||||
else:
|
||||
self.stats.files_modified += 1
|
||||
logger.debug(f"📝 Modified: {file_path}")
|
||||
|
||||
# Track line changes
|
||||
line_count = len(markdown_content.split('\n'))
|
||||
self.stats.total_lines_added += line_count
|
||||
|
||||
return True
|
||||
|
||||
else:
|
||||
logger.warning(f"⚠️ Unknown change type: {change_type}")
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"❌ Error applying file change {file_change.get('file_path')}: {e}")
|
||||
return False
|
||||
|
||||
def _load_section_data(self, section_id: str, public_law_id: str) -> Optional[Dict[str, Any]]:
|
||||
"""Load section data from migrated USC sections"""
|
||||
|
||||
# Try to find section data in USC sections directory
|
||||
sections_dir = Path("data/usc_sections")
|
||||
sections_file = sections_dir / f"{public_law_id}.json"
|
||||
|
||||
if not sections_file.exists():
|
||||
return None
|
||||
|
||||
try:
|
||||
with open(sections_file, 'r') as f:
|
||||
data = json.load(f)
|
||||
sections = data.get('sections', [])
|
||||
|
||||
# Find matching section
|
||||
for section in sections:
|
||||
if section.get('section_id') == section_id:
|
||||
return section
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"❌ Error loading section data for {section_id}: {e}")
|
||||
|
||||
return None
|
||||
|
||||
def _generate_repository_metadata(self, plans_metadata: Dict[str, Any]) -> None:
|
||||
"""Generate comprehensive repository metadata"""
|
||||
|
||||
metadata_dir = self.repo_path / "metadata"
|
||||
|
||||
# Build statistics
|
||||
build_end_time = datetime.now()
|
||||
self.stats.build_duration_seconds = (build_end_time - self.build_start_time).total_seconds()
|
||||
|
||||
# Calculate repository size
|
||||
try:
|
||||
size_result = subprocess.run(['du', '-sm', str(self.repo_path)],
|
||||
capture_output=True, text=True)
|
||||
if size_result.returncode == 0:
|
||||
self.stats.git_repo_size_mb = float(size_result.stdout.split()[0])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Save build statistics
|
||||
stats_data = {
|
||||
"build_completed_at": build_end_time.isoformat(),
|
||||
"build_duration_seconds": self.stats.build_duration_seconds,
|
||||
"build_duration_formatted": str(build_end_time - self.build_start_time),
|
||||
"commits_executed": self.stats.commits_executed,
|
||||
"files_created": self.stats.files_created,
|
||||
"files_modified": self.stats.files_modified,
|
||||
"files_deleted": self.stats.files_deleted,
|
||||
"total_file_operations": self.stats.total_file_operations,
|
||||
"total_lines_added": self.stats.total_lines_added,
|
||||
"git_repo_size_mb": self.stats.git_repo_size_mb,
|
||||
"validation_passed": self.stats.validation_passed,
|
||||
"original_plans_metadata": plans_metadata
|
||||
}
|
||||
|
||||
stats_file = metadata_dir / "build-statistics.json"
|
||||
with open(stats_file, 'w') as f:
|
||||
json.dump(stats_data, f, indent=2, default=str)
|
||||
|
||||
# Create extraction log
|
||||
extraction_log = {
|
||||
"extraction_completed_at": build_end_time.isoformat(),
|
||||
"repository_path": str(self.repo_path),
|
||||
"total_commits": self.stats.commits_executed,
|
||||
"data_sources": {
|
||||
"legal_text": "House Office of Law Revision Counsel",
|
||||
"attribution": "Congress.gov API",
|
||||
"processing": "USC Git Blame System"
|
||||
},
|
||||
"git_repository_info": self._get_git_repository_info()
|
||||
}
|
||||
|
||||
log_file = metadata_dir / "extraction-log.json"
|
||||
with open(log_file, 'w') as f:
|
||||
json.dump(extraction_log, f, indent=2, default=str)
|
||||
|
||||
logger.info("📊 Repository metadata generated")
|
||||
|
||||
def _get_git_repository_info(self) -> Dict[str, Any]:
|
||||
"""Get git repository information"""
|
||||
|
||||
try:
|
||||
# Get commit count
|
||||
commit_count_result = subprocess.run(['git', 'rev-list', '--count', 'HEAD'],
|
||||
cwd=self.repo_path, capture_output=True, text=True)
|
||||
commit_count = int(commit_count_result.stdout.strip()) if commit_count_result.returncode == 0 else 0
|
||||
|
||||
# Get latest commit info
|
||||
latest_commit_result = subprocess.run(['git', 'log', '-1', '--format=%H|%an|%ae|%ad'],
|
||||
cwd=self.repo_path, capture_output=True, text=True)
|
||||
latest_commit_parts = latest_commit_result.stdout.strip().split('|') if latest_commit_result.returncode == 0 else []
|
||||
|
||||
# Get file count
|
||||
file_count_result = subprocess.run(['git', 'ls-files'],
|
||||
cwd=self.repo_path, capture_output=True, text=True)
|
||||
file_count = len(file_count_result.stdout.strip().split('\n')) if file_count_result.returncode == 0 else 0
|
||||
|
||||
return {
|
||||
"commit_count": commit_count,
|
||||
"file_count": file_count,
|
||||
"latest_commit": {
|
||||
"hash": latest_commit_parts[0] if len(latest_commit_parts) > 0 else "",
|
||||
"author": latest_commit_parts[1] if len(latest_commit_parts) > 1 else "",
|
||||
"email": latest_commit_parts[2] if len(latest_commit_parts) > 2 else "",
|
||||
"date": latest_commit_parts[3] if len(latest_commit_parts) > 3 else ""
|
||||
}
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"⚠️ Could not get git repository info: {e}")
|
||||
return {}
|
||||
|
||||
def validate_git_blame(self) -> bool:
|
||||
"""Validate that git blame functionality works correctly"""
|
||||
|
||||
logger.info("🔍 Validating git blame functionality")
|
||||
|
||||
try:
|
||||
# Find markdown files to test
|
||||
md_files = list(self.repo_path.glob("**/*.md"))
|
||||
test_files = [f for f in md_files if f.name != "README.md"][:5] # Test first 5 files
|
||||
|
||||
if not test_files:
|
||||
logger.warning("⚠️ No markdown files found for blame validation")
|
||||
return False
|
||||
|
||||
blame_tests_passed = 0
|
||||
|
||||
for test_file in test_files:
|
||||
try:
|
||||
relative_path = test_file.relative_to(self.repo_path)
|
||||
|
||||
# Run git blame
|
||||
blame_result = subprocess.run(['git', 'blame', str(relative_path)],
|
||||
cwd=self.repo_path, capture_output=True, text=True)
|
||||
|
||||
if blame_result.returncode == 0 and blame_result.stdout:
|
||||
# Check that blame output has proper attribution
|
||||
lines = blame_result.stdout.strip().split('\n')
|
||||
attributed_lines = [line for line in lines if not line.startswith('00000000')]
|
||||
|
||||
if len(attributed_lines) > 0:
|
||||
blame_tests_passed += 1
|
||||
logger.debug(f"✅ Blame test passed: {relative_path}")
|
||||
else:
|
||||
logger.warning(f"⚠️ No attributed lines in: {relative_path}")
|
||||
else:
|
||||
logger.warning(f"⚠️ Blame command failed for: {relative_path}")
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"⚠️ Blame test error for {test_file}: {e}")
|
||||
|
||||
validation_success = blame_tests_passed > 0
|
||||
self.stats.validation_passed = validation_success
|
||||
|
||||
if validation_success:
|
||||
logger.info(f"✅ Git blame validation passed: {blame_tests_passed}/{len(test_files)} files")
|
||||
else:
|
||||
logger.error("❌ Git blame validation failed")
|
||||
|
||||
return validation_success
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"❌ Error during blame validation: {e}")
|
||||
return False
|
||||
|
||||
def _run_git_command(self, args: List[str], description: str) -> None:
|
||||
"""Run a git command with error handling"""
|
||||
|
||||
try:
|
||||
subprocess.run(['git'] + args, cwd=self.repo_path, check=True,
|
||||
capture_output=True, text=True)
|
||||
logger.debug(f"✅ Git command: {description}")
|
||||
except subprocess.CalledProcessError as e:
|
||||
logger.error(f"❌ Git command failed ({description}): {e}")
|
||||
if e.stderr:
|
||||
logger.error(f" Error: {e.stderr}")
|
||||
raise
|
||||
|
||||
def get_build_summary(self) -> Dict[str, Any]:
|
||||
"""Get comprehensive build summary"""
|
||||
|
||||
return {
|
||||
"repository_path": str(self.repo_path),
|
||||
"build_statistics": {
|
||||
"commits_executed": self.stats.commits_executed,
|
||||
"files_created": self.stats.files_created,
|
||||
"files_modified": self.stats.files_modified,
|
||||
"files_deleted": self.stats.files_deleted,
|
||||
"total_file_operations": self.stats.total_file_operations,
|
||||
"total_lines_added": self.stats.total_lines_added,
|
||||
"build_duration_seconds": self.stats.build_duration_seconds,
|
||||
"git_repo_size_mb": self.stats.git_repo_size_mb,
|
||||
"validation_passed": self.stats.validation_passed
|
||||
},
|
||||
"git_info": self._get_git_repository_info()
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
"""Example usage of the git repository builder"""
|
||||
|
||||
# Initialize builder
|
||||
builder = GitRepositoryBuilder(Path("uscode-git-blame"))
|
||||
|
||||
logger.info("🚀 Starting USC git repository build")
|
||||
|
||||
try:
|
||||
# Initialize repository
|
||||
builder.initialize_repository(force=True)
|
||||
|
||||
# Execute commit plans
|
||||
plans_file = Path("data/git_plans/test_commit_sequence.json")
|
||||
if plans_file.exists():
|
||||
builder.execute_commit_plans(plans_file)
|
||||
else:
|
||||
logger.warning(f"⚠️ No commit plans found at {plans_file}")
|
||||
logger.info("ℹ️ Creating minimal test commit...")
|
||||
|
||||
# Create a simple test commit
|
||||
test_file = builder.repo_path / "test-section.md"
|
||||
test_content = """# § 1. Test Section
|
||||
|
||||
This is a test section for demonstrating git blame functionality.
|
||||
|
||||
## Source
|
||||
|
||||
Test source for demonstration purposes.
|
||||
|
||||
---
|
||||
|
||||
**USC Section Metadata:**
|
||||
- Section ID: `test-1-1`
|
||||
- Title: 1
|
||||
- Chapter: 1
|
||||
- Enacted Through: Test
|
||||
|
||||
*Generated by USC Git Blame System*
|
||||
"""
|
||||
test_file.write_text(test_content)
|
||||
|
||||
# Commit the test file
|
||||
builder._run_git_command(['add', '.'], "Add test file")
|
||||
builder._run_git_command(['commit', '-m', 'Add test section for git blame validation'], "Create test commit")
|
||||
|
||||
# Validate git blame functionality
|
||||
validation_success = builder.validate_git_blame()
|
||||
|
||||
# Get build summary
|
||||
summary = builder.get_build_summary()
|
||||
|
||||
# Display results
|
||||
print("\n" + "="*60)
|
||||
print("🏛️ USC GIT REPOSITORY BUILD RESULTS")
|
||||
print("="*60)
|
||||
|
||||
print(f"\nRepository: {summary['repository_path']}")
|
||||
|
||||
stats = summary['build_statistics']
|
||||
print("\nBuild Statistics:")
|
||||
print(f" Commits executed: {stats['commits_executed']}")
|
||||
print(f" Files created: {stats['files_created']}")
|
||||
print(f" Files modified: {stats['files_modified']}")
|
||||
print(f" Files deleted: {stats['files_deleted']}")
|
||||
print(f" Build duration: {stats['build_duration_seconds']:.2f} seconds")
|
||||
print(f" Repository size: {stats['git_repo_size_mb']:.2f} MB")
|
||||
|
||||
git_info = summary['git_info']
|
||||
print("\nGit Repository:")
|
||||
print(f" Total commits: {git_info.get('commit_count', 0)}")
|
||||
print(f" Total files: {git_info.get('file_count', 0)}")
|
||||
|
||||
if validation_success:
|
||||
print("\n✅ Git blame validation: PASSED")
|
||||
print("\nTry these commands:")
|
||||
print(f" cd {builder.repo_path}")
|
||||
print(" git log --oneline")
|
||||
print(" git blame test-section.md")
|
||||
else:
|
||||
print("\n❌ Git blame validation: FAILED")
|
||||
|
||||
print("\n🎉 Repository build complete!")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"❌ Repository build failed: {e}")
|
||||
print(f"\n❌ Build failed: {e}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
338
datastore.py
Normal file
338
datastore.py
Normal file
@@ -0,0 +1,338 @@
|
||||
"""
|
||||
Filesystem-based JSON datastore for US Code git repository system.
|
||||
Provides persistent storage with validation and caching.
|
||||
"""
|
||||
|
||||
import json
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import List, Optional, Dict, Any, Type, TypeVar, Generic
|
||||
from pydantic import BaseModel
|
||||
|
||||
from models import (
|
||||
PublicLaw, Sponsor, Bill, USCodeRelease, CongressionalSession,
|
||||
GitCommitMetadata, APICache, RepositoryMetadata
|
||||
)
|
||||
|
||||
T = TypeVar('T', bound=BaseModel)
|
||||
|
||||
|
||||
class DataStore(Generic[T]):
|
||||
"""Generic filesystem-based datastore for Pydantic models"""
|
||||
|
||||
def __init__(self, model_class: Type[T], base_path: Path, collection_name: str):
|
||||
self.model_class = model_class
|
||||
self.base_path = Path(base_path)
|
||||
self.collection_name = collection_name
|
||||
self.collection_path = self.base_path / collection_name
|
||||
|
||||
# Ensure directory exists
|
||||
self.collection_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Index file for quick lookups
|
||||
self.index_file = self.collection_path / "_index.json"
|
||||
self._index = self._load_index()
|
||||
|
||||
def _load_index(self) -> Dict[str, Dict[str, Any]]:
|
||||
"""Load the index file"""
|
||||
if self.index_file.exists():
|
||||
with open(self.index_file, 'r') as f:
|
||||
return json.load(f)
|
||||
return {}
|
||||
|
||||
def _save_index(self):
|
||||
"""Save the index file"""
|
||||
with open(self.index_file, 'w') as f:
|
||||
json.dump(self._index, f, indent=2, default=str)
|
||||
|
||||
def _get_file_path(self, key: str) -> Path:
|
||||
"""Get file path for a given key"""
|
||||
return self.collection_path / f"{key}.json"
|
||||
|
||||
def save(self, key: str, item: T, metadata: Optional[Dict[str, Any]] = None) -> bool:
|
||||
"""Save an item to the datastore"""
|
||||
try:
|
||||
# Serialize the item
|
||||
item_data = item.dict()
|
||||
|
||||
# Add metadata
|
||||
file_data = {
|
||||
"data": item_data,
|
||||
"metadata": {
|
||||
"saved_at": datetime.now().isoformat(),
|
||||
"model_class": self.model_class.__name__,
|
||||
**(metadata or {})
|
||||
}
|
||||
}
|
||||
|
||||
# Save to file
|
||||
file_path = self._get_file_path(key)
|
||||
with open(file_path, 'w') as f:
|
||||
json.dump(file_data, f, indent=2, default=str)
|
||||
|
||||
# Update index
|
||||
self._index[key] = {
|
||||
"file_path": str(file_path.relative_to(self.base_path)),
|
||||
"model_class": self.model_class.__name__,
|
||||
"saved_at": datetime.now().isoformat(),
|
||||
**(metadata or {})
|
||||
}
|
||||
self._save_index()
|
||||
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
print(f"[!] Error saving {key}: {e}")
|
||||
return False
|
||||
|
||||
def load(self, key: str) -> Optional[T]:
|
||||
"""Load an item from the datastore"""
|
||||
try:
|
||||
file_path = self._get_file_path(key)
|
||||
if not file_path.exists():
|
||||
return None
|
||||
|
||||
with open(file_path, 'r') as f:
|
||||
file_data = json.load(f)
|
||||
|
||||
# Validate and create model instance
|
||||
item_data = file_data.get("data", {})
|
||||
return self.model_class(**item_data)
|
||||
|
||||
except Exception as e:
|
||||
print(f"[!] Error loading {key}: {e}")
|
||||
return None
|
||||
|
||||
def exists(self, key: str) -> bool:
|
||||
"""Check if an item exists"""
|
||||
return key in self._index
|
||||
|
||||
def list_keys(self) -> List[str]:
|
||||
"""List all keys in the datastore"""
|
||||
return list(self._index.keys())
|
||||
|
||||
def delete(self, key: str) -> bool:
|
||||
"""Delete an item from the datastore"""
|
||||
try:
|
||||
file_path = self._get_file_path(key)
|
||||
if file_path.exists():
|
||||
file_path.unlink()
|
||||
|
||||
if key in self._index:
|
||||
del self._index[key]
|
||||
self._save_index()
|
||||
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
print(f"[!] Error deleting {key}: {e}")
|
||||
return False
|
||||
|
||||
def count(self) -> int:
|
||||
"""Count items in the datastore"""
|
||||
return len(self._index)
|
||||
|
||||
def find_by_metadata(self, **filters) -> List[str]:
|
||||
"""Find keys by metadata filters"""
|
||||
matching_keys = []
|
||||
for key, index_entry in self._index.items():
|
||||
match = True
|
||||
for filter_key, filter_value in filters.items():
|
||||
if index_entry.get(filter_key) != filter_value:
|
||||
match = False
|
||||
break
|
||||
if match:
|
||||
matching_keys.append(key)
|
||||
return matching_keys
|
||||
|
||||
|
||||
class USCodeDataStore:
|
||||
"""Main datastore for US Code repository data"""
|
||||
|
||||
def __init__(self, base_path: str = "data"):
|
||||
self.base_path = Path(base_path)
|
||||
self.base_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Initialize individual datastores
|
||||
self.public_laws = DataStore[PublicLaw](PublicLaw, self.base_path, "public_laws")
|
||||
self.sponsors = DataStore[Sponsor](Sponsor, self.base_path, "sponsors")
|
||||
self.bills = DataStore[Bill](Bill, self.base_path, "bills")
|
||||
self.releases = DataStore[USCodeRelease](USCodeRelease, self.base_path, "releases")
|
||||
self.sessions = DataStore[CongressionalSession](CongressionalSession, self.base_path, "sessions")
|
||||
self.commits = DataStore[GitCommitMetadata](GitCommitMetadata, self.base_path, "commits")
|
||||
self.api_cache = DataStore[APICache](APICache, self.base_path, "api_cache")
|
||||
self.metadata = DataStore[RepositoryMetadata](RepositoryMetadata, self.base_path, "metadata")
|
||||
|
||||
# Public Law operations
|
||||
def save_public_law(self, law: PublicLaw) -> bool:
|
||||
"""Save a public law"""
|
||||
key = f"{law.congress}-{law.law_number:03d}"
|
||||
metadata = {
|
||||
"congress": law.congress,
|
||||
"law_number": law.law_number,
|
||||
"enacted_date": law.enacted_date.isoformat()
|
||||
}
|
||||
return self.public_laws.save(key, law, metadata)
|
||||
|
||||
def get_public_law(self, congress: int, law_number: int) -> Optional[PublicLaw]:
|
||||
"""Get a specific public law"""
|
||||
key = f"{congress}-{law_number:03d}"
|
||||
return self.public_laws.load(key)
|
||||
|
||||
def get_public_laws_by_congress(self, congress: int) -> List[PublicLaw]:
|
||||
"""Get all public laws for a congress"""
|
||||
keys = self.public_laws.find_by_metadata(congress=congress)
|
||||
laws = []
|
||||
for key in keys:
|
||||
law = self.public_laws.load(key)
|
||||
if law:
|
||||
laws.append(law)
|
||||
return sorted(laws, key=lambda x: x.law_number)
|
||||
|
||||
# Sponsor operations
|
||||
def save_sponsor(self, sponsor: Sponsor) -> bool:
|
||||
"""Save a sponsor"""
|
||||
chamber_val = sponsor.chamber if isinstance(sponsor.chamber, str) else sponsor.chamber.value
|
||||
party_val = sponsor.party if isinstance(sponsor.party, str) else sponsor.party.value
|
||||
|
||||
key = f"{chamber_val.lower()}_{sponsor.state}_{sponsor.last_name.lower()}_{sponsor.first_name.lower()}"
|
||||
metadata = {
|
||||
"chamber": chamber_val,
|
||||
"state": sponsor.state,
|
||||
"party": party_val,
|
||||
"full_name": sponsor.full_name
|
||||
}
|
||||
return self.sponsors.save(key, sponsor, metadata)
|
||||
|
||||
def find_sponsor_by_name(self, full_name: str) -> Optional[Sponsor]:
|
||||
"""Find a sponsor by full name"""
|
||||
for key in self.sponsors.list_keys():
|
||||
sponsor = self.sponsors.load(key)
|
||||
if sponsor and sponsor.full_name == full_name:
|
||||
return sponsor
|
||||
return None
|
||||
|
||||
# API Cache operations
|
||||
def save_api_cache(self, congress: int, law_number: int, response_data: Dict[str, Any], sponsor: Optional[Sponsor] = None) -> bool:
|
||||
"""Save API cache entry"""
|
||||
cache_key = f"{congress}-{law_number}"
|
||||
cache_entry = APICache(
|
||||
cache_key=cache_key,
|
||||
congress=congress,
|
||||
law_number=law_number,
|
||||
cached_date=datetime.now(),
|
||||
api_response=response_data,
|
||||
sponsor_found=sponsor is not None,
|
||||
sponsor=sponsor
|
||||
)
|
||||
return self.api_cache.save(cache_key, cache_entry)
|
||||
|
||||
def get_api_cache(self, congress: int, law_number: int) -> Optional[APICache]:
|
||||
"""Get cached API response"""
|
||||
cache_key = f"{congress}-{law_number}"
|
||||
return self.api_cache.load(cache_key)
|
||||
|
||||
# US Code Release operations
|
||||
def save_release(self, release: USCodeRelease) -> bool:
|
||||
"""Save a US Code release"""
|
||||
key = f"{release.public_law.congress}-{release.public_law.law_number:03d}"
|
||||
metadata = {
|
||||
"congress": release.public_law.congress,
|
||||
"law_number": release.public_law.law_number,
|
||||
"release_filename": release.release_filename
|
||||
}
|
||||
return self.releases.save(key, release, metadata)
|
||||
|
||||
def get_release(self, congress: int, law_number: int) -> Optional[USCodeRelease]:
|
||||
"""Get a US Code release"""
|
||||
key = f"{congress}-{law_number:03d}"
|
||||
return self.releases.load(key)
|
||||
|
||||
# Git commit operations
|
||||
def save_commit_metadata(self, commit: GitCommitMetadata) -> bool:
|
||||
"""Save git commit metadata"""
|
||||
key = commit.commit_hash[:8] # Use short hash as key
|
||||
metadata = {
|
||||
"congress": commit.public_law.congress,
|
||||
"law_number": commit.public_law.law_number,
|
||||
"commit_date": commit.commit_date.isoformat()
|
||||
}
|
||||
return self.commits.save(key, commit, metadata)
|
||||
|
||||
def get_commits_by_congress(self, congress: int) -> List[GitCommitMetadata]:
|
||||
"""Get all commits for a congress"""
|
||||
keys = self.commits.find_by_metadata(congress=congress)
|
||||
commits = []
|
||||
for key in keys:
|
||||
commit = self.commits.load(key)
|
||||
if commit:
|
||||
commits.append(commit)
|
||||
return sorted(commits, key=lambda x: x.commit_date)
|
||||
|
||||
# Bulk operations
|
||||
def import_house_data(self, house_data_file: Path) -> int:
|
||||
"""Import public laws from House JSON data"""
|
||||
with open(house_data_file, 'r') as f:
|
||||
data = json.load(f)
|
||||
|
||||
imported_count = 0
|
||||
for law_data in data['public_laws']:
|
||||
try:
|
||||
from models import create_public_law_from_house_data
|
||||
law = create_public_law_from_house_data(law_data)
|
||||
if self.save_public_law(law):
|
||||
imported_count += 1
|
||||
except Exception as e:
|
||||
print(f"[!] Error importing law {law_data}: {e}")
|
||||
|
||||
return imported_count
|
||||
|
||||
# Statistics and reporting
|
||||
def get_statistics(self) -> Dict[str, Any]:
|
||||
"""Get datastore statistics"""
|
||||
return {
|
||||
"public_laws": self.public_laws.count(),
|
||||
"sponsors": self.sponsors.count(),
|
||||
"bills": self.bills.count(),
|
||||
"releases": self.releases.count(),
|
||||
"sessions": self.sessions.count(),
|
||||
"commits": self.commits.count(),
|
||||
"api_cache_entries": self.api_cache.count(),
|
||||
"total_files": sum([
|
||||
self.public_laws.count(),
|
||||
self.sponsors.count(),
|
||||
self.bills.count(),
|
||||
self.releases.count(),
|
||||
self.sessions.count(),
|
||||
self.commits.count(),
|
||||
self.api_cache.count()
|
||||
])
|
||||
}
|
||||
|
||||
def validate_integrity(self) -> Dict[str, List[str]]:
|
||||
"""Validate datastore integrity"""
|
||||
issues = {
|
||||
"missing_files": [],
|
||||
"corrupted_files": [],
|
||||
"orphaned_entries": []
|
||||
}
|
||||
|
||||
# Check each datastore
|
||||
for name, datastore in [
|
||||
("public_laws", self.public_laws),
|
||||
("sponsors", self.sponsors),
|
||||
("bills", self.bills),
|
||||
("releases", self.releases),
|
||||
("sessions", self.sessions),
|
||||
("commits", self.commits),
|
||||
("api_cache", self.api_cache)
|
||||
]:
|
||||
for key in datastore.list_keys():
|
||||
try:
|
||||
item = datastore.load(key)
|
||||
if item is None:
|
||||
issues["missing_files"].append(f"{name}/{key}")
|
||||
except Exception:
|
||||
issues["corrupted_files"].append(f"{name}/{key}")
|
||||
|
||||
return issues
|
||||
399
docs/Bills-Summary-XML-User-Guide.md
Normal file
399
docs/Bills-Summary-XML-User-Guide.md
Normal file
@@ -0,0 +1,399 @@
|
||||
U.S. Government Publishing Office Federal Digital System (FDsys) User Guide Document
|
||||
================================================================================
|
||||
|
||||
|
||||
|
||||
## Bill Summaries XML Bulk Data
|
||||
|
||||
Prepared by: Programs, Strategy, and Technology
|
||||
|
||||
U.S. Government Publishing Office
|
||||
|
||||
January 2015
|
||||
|
||||
|
||||
## Revision
|
||||
|
||||
- 1.0 January 2014 Version 1.0
|
||||
House Bill Summaries
|
||||
- 2.0 January 2015 Version 2.0
|
||||
House and Senate Bill Summaries
|
||||
|
||||
|
||||
## 1. Introduction
|
||||
|
||||
At the direction of the Appropriations Committee within the United States House of
|
||||
Representatives, in support of the Legislative Branch Bulk Data Task Force, the Government
|
||||
Publishing Office (GPO), the Library of Congress (LOC), the Clerk of the House, and the Secretary
|
||||
of the Senate are making Bill Summaries in XML format available through the GPO’s Federal
|
||||
Digital System (FDsys) Bulk Data repository starting with the 113th Congress. The FDsys Bulk
|
||||
Data repository for Bill Summaries is available at
|
||||
<http://www.gpo.gov/fdsys/bulkdata/BILLSUM>.
|
||||
|
||||
|
||||
### 1.1 Types of Bill Summaries
|
||||
|
||||
Bill summaries are summaries of bills or resolutions, as well as other document types associated
|
||||
with the legislative history of a measure such as amendments, committee reports, conference
|
||||
reports, or public laws (enacted bills or joint resolutions). A bill summary describes the most
|
||||
significant provisions of a piece of legislation and details the effects the legislative text may have on
|
||||
current law and Federal programs. Bill summaries are written as a result of a Congressional action
|
||||
and may not always map to a printed bill version. Bill summaries are authored by the Congressional
|
||||
Research Service (CRS) of the Library of Congress. As stated in Public Law 91-510 (2 USC 166
|
||||
(d)(6)), one of the duties of CRS is "to prepare summaries and digests of bills and resolutions of a
|
||||
public general nature introduced in the Senate or House of Representatives.”
|
||||
|
||||
|
||||
#### Bills
|
||||
|
||||
- House Bill (HR)
|
||||
- Senate Bill (S)
|
||||
|
||||
A bill is a legislative proposal before Congress. Bills from each house are assigned a number in
|
||||
the order in which they are introduced, starting at the beginning of each Congress (first and
|
||||
second sessions). Public bills pertain to matters that affect the general public or classes of
|
||||
citizens, while private bills pertain to individual matters that affect individuals and organizations,
|
||||
such as claims against the Government.
|
||||
|
||||
|
||||
#### Joint Resolutions
|
||||
|
||||
- House Joint Resolution (HJRES)
|
||||
- Senate Joint Resolution (SJRES)
|
||||
|
||||
A joint resolution is a legislative proposal that requires the approval of both houses and the
|
||||
signature of the President, just as a bill does. Resolutions from each house are assigned a number
|
||||
in the order in which they are introduced, starting at the beginning of each Congress (first and
|
||||
second sessions). There is no real difference between a bill and a joint resolution. Joint
|
||||
resolutions generally are used for limited matters, such as a single appropriation for a specific
|
||||
purpose. They are also used to propose amendments to the Constitution.
|
||||
1A joint resolution has the force of law, if approved. Joint resolutions become a part of the
|
||||
Constitution when three-quarters of the states have ratified them; they do not require the
|
||||
President's signature.
|
||||
|
||||
|
||||
#### Concurrent Resolutions
|
||||
|
||||
- House Concurrent Resolution (HCONRES)
|
||||
- Senate Concurrent Resolution (SCONRES)
|
||||
|
||||
A concurrent resolution is a legislative proposal that requires the approval of both houses but
|
||||
does not require the signature of the President and does not have the force of law. Concurrent
|
||||
resolutions generally are used to make or amend rules that apply to both houses. They are also
|
||||
used to express the sentiments of both of the houses. For example, a concurrent resolution is used
|
||||
to set the time of Congress' adjournment. It may also be used by Congress to convey
|
||||
congratulations to another country on the anniversary of its independence.
|
||||
|
||||
|
||||
#### Simple Resolutions
|
||||
|
||||
- House Simple Resolution (HRES)
|
||||
- Senate Simple Resolution (SRES)
|
||||
|
||||
A simple resolution is a legislative proposal that addresses matters entirely within the prerogative
|
||||
of one house or the other. It requires neither the approval of the other house nor the signature of
|
||||
the President, and it does not have the force of law. Most simple resolutions concern the rules of
|
||||
one house. They are also used to express the sentiments of a single house. For example, a simple
|
||||
resolution may offer condolences to the family of a deceased member of Congress, or it may
|
||||
express the sense of the Senate or House on foreign policy or other executive business.
|
||||
|
||||
Additional information about bill types can be found at
|
||||
<http://www.gpo.gov/help/index.html#about_congressional_bills.htm>.
|
||||
|
||||
|
||||
### 1.2 Scope of Bulk Data
|
||||
|
||||
The Bill Summaries bulk data collection on FDsys includes XML bills summaries from the 113 th
|
||||
Congress forward.
|
||||
|
||||
|
||||
### 1.3 Bulk Data Downloads
|
||||
|
||||
The Bulk Data repository is organized by Congress and bill type. A ZIP file is available for each
|
||||
bill type and contains Bill Summaries XML files for that bill type within a specific Congress.
|
||||
Each Bill Summaries XML file contains summaries of legislation under consideration for a
|
||||
specific measure.
|
||||
|
||||
|
||||
## 2. XML Descriptions
|
||||
|
||||
The following conventions are used in this document:
|
||||
|
||||
- XML element names are denoted with angled brackets and in courier. For example,
|
||||
`<title>` is an XML element.
|
||||
- XML attribute names are denoted with a “@” prefix and in courier. For example, @href
|
||||
is an XML attribute.
|
||||
|
||||
|
||||
### 2.1 Elements
|
||||
|
||||
- `<BillSummaries>`
|
||||
|
||||
Root element.
|
||||
|
||||
- `<item>`
|
||||
|
||||
Parent container for a single legislative measure.
|
||||
- `<title>`
|
||||
|
||||
The latest title for the measure. It may be the official title or the short
|
||||
title. It is contained within `<item>`.
|
||||
|
||||
- `<summary>`
|
||||
|
||||
Parent container for a single summary. It may appear one or more times in
|
||||
the file. It is contained within `<item>`.
|
||||
|
||||
- `<action-date>`
|
||||
|
||||
The date on which a particular action occurred. The format is YYYY‐MM‐DD. It
|
||||
is contained within `<summary>`.
|
||||
|
||||
- `<action-desc>`
|
||||
|
||||
The description of the action that took place to prompt the bill summary to
|
||||
be written. It is contained within `<summary>`. This value is added by CRS.
|
||||
|
||||
See Section 3 of this document for a list of possible values.
|
||||
|
||||
- `<summary-text>`
|
||||
|
||||
This is the text of the summary written by the Congressional Research Service
|
||||
of the Library of Congress. It is contained within `<summary>`. Values are
|
||||
enclosed in a CDATA tag and contain HTML elements.
|
||||
|
||||
|
||||
### 2.2 Attributes
|
||||
|
||||
- `@congress`
|
||||
|
||||
The number of the Congress. This is an attribute of `<item>`.
|
||||
|
||||
- `@measure-type`
|
||||
|
||||
The type of measure. This is an attribute of `<item>`. The measure type
|
||||
abbreviations that can be found in bill summaries are hr, hjres, hconres,
|
||||
hres, s, sconres, sres, and sjres.
|
||||
|
||||
See Section 1.1 of this document for a description of each measure type.
|
||||
|
||||
- `@measure-number`
|
||||
|
||||
The number associated with the measure. This is commonly referred to as the
|
||||
bill number. This is an attribute of `<item>`.
|
||||
|
||||
- `@measure-id`
|
||||
|
||||
An ID assigned to the measure. This is an attribute of `<item>`.
|
||||
|
||||
Convention: “id” + Congress number + measure type abbreviation + measure
|
||||
number
|
||||
|
||||
Example: id113hr910
|
||||
|
||||
The measure type abbreviations that can be found in bill summaries are
|
||||
hr, hjres, hconres, hres, s, sconres, sres, and sjres.
|
||||
|
||||
See Section 1.1 of this document for a description of each measure type.
|
||||
|
||||
- `@originChamber`
|
||||
|
||||
The chamber in which the measure originated. This is an attribute of
|
||||
`<item>`. Value will be HOUSE or SENATE.
|
||||
|
||||
- `@orig-publish-date`
|
||||
|
||||
The first date in which the bill summary file was published. The format is
|
||||
YYYY‐MM‐DD. This is an attribute of `<item>`.
|
||||
|
||||
- `@update-date`
|
||||
|
||||
The date in which the material in the container element was last updated. The
|
||||
format is YYYY‐MM‐DD. This is an attribute of `<item>` and `<summary>`.
|
||||
|
||||
- `@summary-id`
|
||||
|
||||
An ID assigned to the individual summary. This is an attribute of `<summary>`.
|
||||
|
||||
Convention: “id” + Congress number + measure type abbreviation + measure
|
||||
cumber + the letter “v” for version + LOC action code for summaries
|
||||
|
||||
Example: id113hr910v28
|
||||
|
||||
The measure type abbreviations that can be found in bill summaries are hr,
|
||||
hjres, hconres, hres, s, sconres, sres, and sjres.
|
||||
|
||||
See Section 3 of this document for a list of LOC action codes for summaries.
|
||||
|
||||
- `@currentChamber`
|
||||
|
||||
The chamber in which the action described in the `<action-desc>` element
|
||||
occurred. This is an attribute of `<summary>`. Value will be HOUSE, SENATE,
|
||||
or BOTH.
|
||||
|
||||
|
||||
### 2.3. Sample Bill Summaries XML File
|
||||
|
||||
```
|
||||
<BillSummaries>
|
||||
<item congress="113" measure-type="hr" measure-number="910" measure-
|
||||
id="id113hr910" originChamber="HOUSE" orig-publish-date="2013-02-28"
|
||||
update-date="2013-09-03">
|
||||
<title>Sikes Act Reauthorization Act of 2013</title>
|
||||
<summary summary-id="id113hr910v28" currentChamber="HOUSE" update-
|
||||
date="2013-09-03">
|
||||
<action-date>2013-06-24</action-date>
|
||||
<action-desc>Reported to House without amendment, Part I</action-desc>
|
||||
<summary-text><![CDATA[ <p><b>Sikes Act Reauthorization Act of 2013 - Reauthorizes title I
|
||||
of the Sikes Act (conservation programs on military installations) for FY2015-
|
||||
FY2019.</b></p>]]></summary-text>
|
||||
</summary>
|
||||
<summary summary-id="id113hr910v00" currentChamber="HOUSE" update-
|
||||
date="2013-02-28">
|
||||
<action-date>2013-02-28</action-date>
|
||||
<action-desc>Introduced in House</action-desc>
|
||||
<summary-text><![CDATA[<p><b>(This measure has not been amended since it was introduced.
|
||||
The summary of that version is repeated here.)</b></p> <p>Sikes Act Reauthorization Act of 2013 -
|
||||
Reauthorizes title I of the Sikes Act (conservation programs on military installations) for FY2015-
|
||||
FY2019.</p>]]></summary-text>
|
||||
</summary>
|
||||
</item>
|
||||
<dublinCore xmlns:dc=”http://purl.org/dc/elements/1.1/”>
|
||||
<dc:format>text/xml</dc:format>
|
||||
<dc:language>EN</dc:language>
|
||||
<dc:rights>Pursuant to Title 17 Section 105 of the United States Code, this file is not subject to
|
||||
copyright protection and is in the public domain.</dc:rights>
|
||||
<dc:contributor>Congressional Research Service, Library of Congress</dc:contributor>
|
||||
<dc:description>This file contains bill summaries for federal legislation. A bill summary describes
|
||||
the most significant provisions of a piece of legislation and details the effects the legislative text may
|
||||
have on current law and federal programs. Bill summaries are authored by the Congressional Research
|
||||
Service (CRS) of the Library of Congress. As stated in Public Law 91-510 (2 USC 166 (d)(6)), one of the
|
||||
duties of CRS is "to prepare summaries and digests of bills and resolutions of a public general nature
|
||||
introduced in the Senate or House of Representatives". For more information, refer to the User Guide that
|
||||
accompanies this file.</dc:description>
|
||||
</dublinCore>
|
||||
</BillSummaries>
|
||||
```
|
||||
|
||||
|
||||
## 3. Mapping of LOC Action Codes, Action Description Text, and Version Codes
|
||||
|
||||
LOC Action Code for Summaries| Chamber | Text in the `<action‐desc>` Element | LOC Version Code
|
||||
-----------------------------|---------|-----------------------------------------------------------------|-----------------
|
||||
00 | HOUSE | Introduced in House | IH
|
||||
00 | SENATE | Introduced in Senate | IS
|
||||
01 | SENATE | Reported to Senate amended | RS
|
||||
02 | SENATE | Reported to Senate amended, 1st committee reporting | RS
|
||||
03 | SENATE | Reported to Senate amended, 2nd committee reporting | RS
|
||||
04 | SENATE | Reported to Senate amended, 3rd committee reporting | RS
|
||||
05 | SENATE | Reported to Senate amended, 4th committee reporting | RS
|
||||
06 | SENATE | Reported to Senate amended, 5th committee reporting | RS
|
||||
07 | SENATE | Reported to Senate amended, 6th committee reporting | RS
|
||||
08 | SENATE | Reported to Senate amended, 7th committee reporting | RS
|
||||
09 | SENATE | Reported to Senate amended, 8th committee reporting | RS
|
||||
10 | SENATE | Reported to Senate amended, 9th committee reporting | RS
|
||||
11 | SENATE | Reported to Senate amended, 10th committee reporting | RS
|
||||
12 | SENATE | Reported to Senate without amendment, 1st committee reporting | RS
|
||||
13 | SENATE | Reported to Senate without amendment, 2nd committee reporting | RS
|
||||
14 | SENATE | Reported to Senate without amendment, 3rd committee reporting | RS
|
||||
15 | SENATE | Reported to Senate without amendment, 4th committee reporting | RS
|
||||
16 | SENATE | Reported to Senate without amendment, 5th committee reporting | RS
|
||||
17 | HOUSE | Reported to House amended | RH
|
||||
18 | HOUSE | Reported to House amended, Part I | RH
|
||||
19 | HOUSE | Reported to House amended, Part II | RH
|
||||
20 | HOUSE | Reported to House amended, Part III | RH
|
||||
21 | HOUSE | Reported to House amended, Part IV | RH
|
||||
22 | HOUSE | Reported to House amended, Part V | RH
|
||||
23 | HOUSE | Reported to House amended, Part VI | RH
|
||||
24 | HOUSE | Reported to House amended, Part VII | RH
|
||||
25 | HOUSE | Reported to House amended, Part VIII | RH
|
||||
26 | HOUSE | Reported to House amended, Part IX | RH
|
||||
27 | HOUSE | Reported to House amended, Part X | RH
|
||||
28 | HOUSE | Reported to House without amendment, Part I | RH
|
||||
29 | HOUSE | Reported to House without amendment, Part II | RH
|
||||
30 | HOUSE | Reported to House without amendment, Part III | RH
|
||||
31 | HOUSE | Reported to House without amendment, Part IV | RH
|
||||
32 | HOUSE | Reported to House without amendment, Part V | RH
|
||||
33 | HOUSE | Laid on table in House | LTH
|
||||
34 | SENATE | Indefinitely postponed in Senate | IPS
|
||||
35 | SENATE | Passed Senate amended | ES
|
||||
36 | HOUSE | Passed House amended | EH
|
||||
37 | SENATE | Failed of passage in Senate | FPS
|
||||
38 | HOUSE | Failed of passage in House | FPH
|
||||
39 | HOUSE | Senate agreed to House amendment with amendment | ATS
|
||||
40 | SENATE | House agreed to Senate amendment with amendment | ATH
|
||||
41 | HOUSE | Senate disagreed to House amendment with amendment | NAT
|
||||
42 | SENATE | House disagreed to Senate amendment with amendment | NAT
|
||||
43 | HOUSE | Senate disagreed to House amendment | NAT
|
||||
44 | SENATE | House disagreed to Senate amendment | NAT
|
||||
45 | SENATE | Senate receded and concurred with amendment | AES
|
||||
46 | HOUSE | House receded and concurred with amendment | EAH
|
||||
47 | SENATE | Conference report filed in Senate | CONF-S
|
||||
48 | HOUSE | Conference report filed in House | CONF-H
|
||||
49 | BOTH | Public Law | LAW
|
||||
50 | BOTH | Private Law | LAW
|
||||
51 | BOTH | Line item veto by President | LINEITEMVETO
|
||||
52 | SENATE | Passed Senate amended, 2nd occurrence | ES
|
||||
53 | SENATE | Passed Senate amended, 3rd occurrence | ES
|
||||
54 | HOUSE | Passed House amended, 2nd occurrence | EH
|
||||
55 | HOUSE | Passed House amended, 3rd occurrence | EH
|
||||
56 | SENATE | Senate vitiated passage of bill after amendment | PAV
|
||||
57 | HOUSE | House vitiated passage of bill after amendment | PAV
|
||||
58 | SENATE | Motion to recommit bill as amended in Senate | MOTION_R-S
|
||||
59 | HOUSE | Motion to recommit bill as amended in House | MOTION_R-H
|
||||
60 | SENATE | Senate agreed to House amendment with amendment, 2nd occurrence | ATS
|
||||
61 | SENATE | Senate agreed to House amendment with amendment, 3rd occurrence | ATS
|
||||
62 | HOUSE | House agreed to Senate amendment with amendment, 2nd occurrence | ATH
|
||||
63 | HOUSE | House agreed to Senate amendment with amendment, 3rd occurrence | ATH
|
||||
64 | SENATE | Senate receded and concurred with amendment, 2nd occurrence | AES
|
||||
65 | SENATE | Senate receded and concurred with amendment, 3rd occurrence | AES
|
||||
66 | HOUSE | House receded and concurred with amendment, 2nd occurrence | EAH
|
||||
67 | HOUSE | House receded and concurred with amendment, 3rd occurrence | EAH
|
||||
70 | HOUSE | Hearing scheduled in House | HRG-SCD-H
|
||||
71 | SENATE | Hearing scheduled in Senate | HRG-SCD-S
|
||||
72 | HOUSE | Hearing held in House | HRG-H
|
||||
73 | SENATE | Hearing held in Senate | HRG-S
|
||||
74 | HOUSE | Markup in House | MKUP-H
|
||||
75 | SENATE | Markup in Senate | MKUP-S
|
||||
76 | HOUSE | Rule reported to House | RULE-H
|
||||
77 | HOUSE | Discharged from House committee | CDH
|
||||
78 | SENATE | Discharged from Senate committee | CDS
|
||||
79 | HOUSE | Reported to House, without amendment | RH
|
||||
80 | SENATE | Reported to Senate without amendment | RS
|
||||
81 | HOUSE | Passed House, without amendment | EH
|
||||
82 | SENATE | Passed Senate, without amendment | ES
|
||||
83 | SENATE | Conference report filed in Senate, 2nd conference report | CONF-S
|
||||
84 | SENATE | Conference report filed in Senate, 3rd conference report | CONF-S
|
||||
85 | SENATE | Conference report filed in Senate, 4th conference report | CONF-S
|
||||
86 | HOUSE | Conference report filed in House, 2nd conference report | CONF-H
|
||||
87 | HOUSE | Conference report filed in House, 3rd conference report | CONF-H
|
||||
88 | HOUSE | Conference report filed in House, 4th conference report | CONF-H
|
||||
|
||||
|
||||
## 4. Data Set
|
||||
|
||||
Bill Summaries data is provided to GPO by the Library of Congress, and XML files are available
|
||||
for bulk data download on the FDsys Bulk Data repository starting with the 113th Congress
|
||||
(2013-2014). Bill Summaries XML files are not available through FDsys search or browse; they
|
||||
are only available in the FDsys Bulk Data repository.
|
||||
|
||||
In general, there are no restrictions on re-use of information in the Bill Summaries data set
|
||||
because U.S. Government works are not subject to copyright protection and are in the public
|
||||
domain. GPO and its legislative branch data partners do not restrict downstream uses of Bill
|
||||
Summaries data, except that independent providers should be aware that only GPO and its
|
||||
legislative branch data partners are entitled to represent that they are the providers of official Bill
|
||||
Summaries data.
|
||||
|
||||
Bill Summaries XML files can be manipulated and enriched to operate in the various
|
||||
applications that users may devise. GPO and its legislative branch data partners cannot vouch for
|
||||
the authenticity of data that is not under GPO’s control. GPO is providing free access to Bill
|
||||
Summaries XML files for display in various applications and mash-ups outside the FDsys
|
||||
domain. GPO does not endorse third party applications, and does not evaluate how the original
|
||||
legal content is displayed on other sites. Consumers should form their own conclusions as to
|
||||
whether the downloaded data can be relied upon within an application or mash-up.
|
||||
|
||||
|
||||
## 5. Resources Directory
|
||||
|
||||
The resources directory at <http://www.gpo.gov/fdsys/bulkdata/BILLSUM/resources>
|
||||
contains the *User Guide for Bill Summaries XML Bulk Data* in PDF form.
|
||||
159
docs/Bills-XML-User-Guide.md
Normal file
159
docs/Bills-XML-User-Guide.md
Normal file
@@ -0,0 +1,159 @@
|
||||
U.S. Government Publishing Office Federal Digital System (FDsys) User Guide Document
|
||||
==================================================================================
|
||||
|
||||
## Bills XML Bulk Data
|
||||
|
||||
Prepared by: Programs, Strategy and Technology
|
||||
|
||||
U.S. Government Printing Office
|
||||
|
||||
January 2015
|
||||
|
||||
### Revision History
|
||||
|
||||
- 1.0 December 2012 Version 1.0
|
||||
House Bills
|
||||
- 2.0 January 2015 Version 2.0
|
||||
House and Senate Bills
|
||||
|
||||
## Introduction
|
||||
|
||||
At the direction of the Appropriations Committee within the United States House of
|
||||
Representatives, in support of the Legislative Branch Bulk Data Task Force, the Government
|
||||
Printing Office (GPO), the Library of Congress (LOC), the Clerk of the House, and the Secretary
|
||||
of the Senate are making bills in XML format available through the GPO’s Federal Digital
|
||||
System (FDsys) Bulk Data repository starting with the 113th Congress. The FDsys Bulk Data
|
||||
repository for bills is available at <http://www.gpo.gov/fdsys/bulkdata/BILLS>. Please see FDsys
|
||||
at <http://www.fdsys.gov> for access to individual House and Senate Congressional Bills in PDF
|
||||
and HTML formats.
|
||||
|
||||
### Types of Legislation
|
||||
|
||||
Four types of legislation are available on the Bulk Data repository. This section provides a brief
|
||||
overview of each type of legislation.
|
||||
|
||||
#### Bills
|
||||
|
||||
- House Bill (HR)
|
||||
- Senate Bill (S)
|
||||
|
||||
A bill is a legislative proposal before Congress. Bills from each house are assigned a number in
|
||||
the order in which they are introduced, starting at the beginning of each Congress (first and
|
||||
second sessions). Public bills pertain to matters that affect the general public or classes of
|
||||
citizens, while private bills pertain to individual matters that affect individuals and organizations,
|
||||
such as claims against the Government.
|
||||
|
||||
#### Joint Resolutions
|
||||
|
||||
- House Joint Resolution (HJRES)
|
||||
- Senate Joint Resolution (SJRES)
|
||||
|
||||
A joint resolution is a legislative proposal that requires the approval of both houses and the
|
||||
signature of the President, just as a bill does. Resolutions from each house are assigned a number
|
||||
in the order in which they are introduced, starting at the beginning of each Congress (first and
|
||||
second sessions). There is no real difference between a bill and a joint resolution. Joint
|
||||
resolutions generally are used for limited matters, such as a single appropriation for a specific
|
||||
purpose. They are also used to propose amendments to the Constitution. A joint resolution has
|
||||
the force of law, if approved. Joint resolutions become a part of the Constitution when three-
|
||||
quarters of the states have ratified them; they do not require the President's signature.
|
||||
|
||||
|
||||
#### Concurrent Resolutions
|
||||
|
||||
- House Concurrent Resolution (HCONRES)
|
||||
- Senate Concurrent Resolution (SCONRES)
|
||||
|
||||
A concurrent resolution is a legislative proposal that requires the approval of both houses but
|
||||
does not require the signature of the President and does not have the force of law. Concurrent
|
||||
resolutions generally are used to make or amend rules that apply to both houses. They are also
|
||||
used to express the sentiments of both of the houses. For example, a concurrent resolution is used
|
||||
to set the time of Congress' adjournment. It may also be used by Congress to convey
|
||||
congratulations to another country on the anniversary of its independence.
|
||||
|
||||
#### Simple Resolutions
|
||||
|
||||
- House Simple Resolution (HRES)
|
||||
- Senate Simple Resolution (SRES)
|
||||
|
||||
A simple resolution is a legislative proposal that addresses matters entirely within the prerogative
|
||||
of one house or the other. It requires neither the approval of the other house nor the signature of
|
||||
the President, and it does not have the force of law. Most simple resolutions concern the rules of
|
||||
one house. They are also used to express the sentiments of a single house. For example, a simple
|
||||
resolution may offer condolences to the family of a deceased member of Congress, or it may
|
||||
give "advice" on foreign policy or other executive business.
|
||||
|
||||
Additional information about bill types and versions is available at
|
||||
<http://www.gpo.gov/help/index.html#about_congressional_bills.htm>.
|
||||
|
||||
### Scope of Bulk Data
|
||||
|
||||
The Bills data collection on FDsys includes XML bill texts from the 113 th Congress forward.
|
||||
|
||||
### Bulk Data Downloads
|
||||
|
||||
The Bulk Data repository is organized by Congress, session, and bill type. A ZIP file is available
|
||||
for each bill type and contains all bill XML files for that bill type within a specific session and
|
||||
Congress.
|
||||
|
||||
## Authenticity of Bill XML Files
|
||||
|
||||
### Q. What is the data set available for bills in XML?
|
||||
|
||||
A. Bill files in XML are provided to GPO by the House of Representatives and Senate and are
|
||||
available starting in 2013 with the 113th Congress.
|
||||
|
||||
### Q. How do the bulk XML files offered on the FDsys Bulk Data repository relate to the digitally signed PDF files available on FDsys?
|
||||
|
||||
A. GPO makes Congressional Bills from the 103rd Congress forward available on FDsys in
|
||||
digitally signed PDF and HTML formats. Generally, House and Senate bills from the 111th
|
||||
Congress forward are also available in XML on FDsys.
|
||||
|
||||
### Q. What does the term “digitally signed” mean?
|
||||
|
||||
A. Currently, GPO uses digital signature technology on PDF documents to add a visible Seal of
|
||||
Authenticity (a graphic of an eagle) to authenticated and certified documents. The technology
|
||||
allows GPO to assure data integrity, and provide users with assurance that the content is
|
||||
unchanged since it was disseminated by GPO. A signed and certified document also displays a
|
||||
blue ribbon icon near the Seal of Authenticity and in the Signatures tab within Adobe Acrobat or
|
||||
Reader. When users print a document that has been signed and certified by GPO, the Seal of
|
||||
Authenticity will automatically print on the document, but the blue ribbon will not print.
|
||||
|
||||
### Q. Are bill XML bulk data download files digitally signed?
|
||||
|
||||
A. No, XML files available for individual or bulk download are not digitally signed. They can be
|
||||
manipulated and enriched to operate in the various applications that users may devise. GPO is
|
||||
evaluating technology that could be used to digitally sign XML files on FDsys. Adding signed
|
||||
non-PDF files to FDsys would be an enhancement for FDsys users, but would not be used to
|
||||
restrict or adversely affect the XML bulk data downloads. The integrity of a bill XML file can be
|
||||
verified by checking its SHA-256 hash value against the hash value recorded in the PREMIS
|
||||
metadata file for each bill on FDsys.
|
||||
|
||||
### Q. What is the authenticity of bill XML files after they have been downloaded to another site?
|
||||
|
||||
A. We cannot vouch for the authenticity of data that is not under GPO’s control. GPO is
|
||||
providing free access to bill data via XML for display in various applications and mash-ups
|
||||
outside the FDsys domain. GPO does not endorse third party applications, and does not evaluate
|
||||
how our original legal content is displayed on other sites. Consumers should form their own
|
||||
conclusions as to whether the downloaded data can be relied upon within an application or mash-
|
||||
up. An application may link to the official bill files on FDsys to provide users with additional
|
||||
assurance. The authenticated digitally-signed PDF is available on FDsys at
|
||||
<http://www.fdsys.gov>.
|
||||
|
||||
### Q. Does GPO assert any control over downstream uses of bulk data?
|
||||
|
||||
A. In general, there are no restrictions on re-use of information in bills because U.S.
|
||||
Government works are not subject to copyright. GPO does not restrict downstream uses of bill
|
||||
data, except that independent providers should be aware that only GPO and its legislative
|
||||
branch data partners are entitled to represent that they are the providers of the official versions of
|
||||
bills.
|
||||
|
||||
### Q. How can re-publishers indicate the source of a bill XML file?
|
||||
|
||||
A. Re-publishers of bills in XML may cite FDsys and GPO as the source of their data, and they are free to characterize the quality of data as it appears on their site.
|
||||
|
||||
## Resources Directory
|
||||
|
||||
The resources directory at <http://www.gpo.gov/fdsys/bulkdata/BILLS/resources> contains the
|
||||
current version of the DTD, stylesheets, and associated graphics which, when placed in the same
|
||||
directory as a bill XML file, are used to display the XML file in a browser. Additional
|
||||
information about bills in XML can be found at <http://xml.house.gov>.
|
||||
116
download_cache.py
Normal file
116
download_cache.py
Normal file
@@ -0,0 +1,116 @@
|
||||
|
||||
import json
|
||||
from venv import logger
|
||||
import sys
|
||||
import requests
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
GOV_BULK_SITE="https://www.govinfo.gov/bulkdata"
|
||||
CACHE_DIR = "cache"
|
||||
|
||||
def scrape(page: str, cache_dir: Path):
|
||||
"""
|
||||
Main entry point for the scraping process.
|
||||
This function orchestrates the scraping of various data sources.
|
||||
"""
|
||||
|
||||
# Get page.
|
||||
cached_page = cache_dir / "page.json"
|
||||
if cached_page.exists():
|
||||
with open(cached_page, 'r', encoding='utf-8') as f:
|
||||
body = json.load(f)
|
||||
else:
|
||||
cache_dir.mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
response = requests.get(page, headers={"User-Agent": "GitLawScraper/1.0", "Accept": "application/json"}, timeout=30)
|
||||
response.raise_for_status()
|
||||
if 'application/json' in response.headers.get('Content-Type', ''):
|
||||
body = response.json()
|
||||
with open(cached_page, 'w', encoding='utf-8') as f:
|
||||
json.dump(body, f)
|
||||
else:
|
||||
print(f"Non-JSON response from {page}")
|
||||
return
|
||||
print(f"Cached resource: {page}")
|
||||
except requests.RequestException as e:
|
||||
print(f"❌ Failed to fetch resource {page}: {e}")
|
||||
return
|
||||
|
||||
files: list[dict[str, Any]] = body.get('files', [])
|
||||
|
||||
# Look for a zip file if we're in a new directory.
|
||||
in_new_dir = len(list(cache_dir.glob('*'))) == 1
|
||||
if in_new_dir:
|
||||
zip_file = next((f for f in files if f.get("mimeType") == "application/zip"), None)
|
||||
if zip_file:
|
||||
print(f"📦 Downloading zip file: {zip_file['link']}")
|
||||
zip_url = zip_file.get('link')
|
||||
if zip_url:
|
||||
try:
|
||||
# Download the zip file.
|
||||
response = requests.get(zip_url, headers={"User-Agent": "GitLawScraper/1.0"}, timeout=30)
|
||||
response.raise_for_status()
|
||||
zip_path = cache_dir / zip_file['justFileName']
|
||||
with open(zip_path, 'wb') as f:
|
||||
f.write(response.content)
|
||||
print(f"✅ Downloaded zip file: {zip_file['link']}")
|
||||
|
||||
# Unzip the file.
|
||||
import zipfile
|
||||
with zipfile.ZipFile(zip_path, 'r') as zip_ref:
|
||||
zip_ref.extractall(cache_dir)
|
||||
print(f"✅ Unzipped files to {cache_dir}")
|
||||
|
||||
except requests.RequestException as e:
|
||||
print(f"❌ Failed to download zip file {zip_file['justFileName']}: {e}")
|
||||
else:
|
||||
print("No zip file found, continuing with individual files.")
|
||||
|
||||
for file in files:
|
||||
|
||||
# Download non-folder files directly.
|
||||
if not file.get("folder", False):
|
||||
url = file.get('link')
|
||||
if url:
|
||||
|
||||
file_path = cache_dir / file['justFileName']
|
||||
if file_path.exists():
|
||||
print(f"✅ File already exists: {file['justFileName']}")
|
||||
continue
|
||||
|
||||
print(f"📥 Downloading file: {file['justFileName']} from {url}")
|
||||
try:
|
||||
response = requests.get(url, headers={"User-Agent": "GitLawScraper/1.0"}, timeout=30)
|
||||
response.raise_for_status()
|
||||
with open(file_path, 'wb') as f:
|
||||
f.write(response.content)
|
||||
print(f"✅ Downloaded file: {file['justFileName']}")
|
||||
except requests.RequestException as e:
|
||||
print(f"❌ Failed to download file {file['justFileName']}: {e}")
|
||||
continue
|
||||
|
||||
# Recursively scrape folders.
|
||||
scrape(file['link'], cache_dir / file['justFileName'])
|
||||
|
||||
def main():
|
||||
print("🚀 Starting scraping process for US Code data...")
|
||||
cache_dir = Path(CACHE_DIR)
|
||||
if not cache_dir.exists():
|
||||
cache_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
try:
|
||||
scrape("https://www.govinfo.gov/bulkdata/json/BILLS", cache_dir / "BILLS")
|
||||
scrape("https://www.govinfo.gov/bulkdata/json/BILLSTATUS", cache_dir / "BILLSTATUS")
|
||||
scrape("https://www.govinfo.gov/bulkdata/json/BILLSUM", cache_dir / "BILLSUM")
|
||||
scrape("https://www.govinfo.gov/bulkdata/json/PLAW", cache_dir / "PLAW" )
|
||||
scrape("https://www.govinfo.gov/bulkdata/json/STATUTES", cache_dir / "STATUTES" )
|
||||
except Exception as e:
|
||||
logger.error(f"❌ An error occurred during scraping: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
print("🎉 Scraping completed without errors")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
735
generate_git_plan.py
Normal file
735
generate_git_plan.py
Normal file
@@ -0,0 +1,735 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
USC Git Blame Commit Plan Generator
|
||||
|
||||
Analyzes migrated data to create intelligent incremental git commit plans:
|
||||
|
||||
1. Compares USC releases to identify section-level changes
|
||||
2. Maps changes to specific public laws and sponsors
|
||||
3. Generates optimized commit sequences for proper git blame
|
||||
4. Creates comprehensive commit plans with rich attribution
|
||||
5. Validates chronological ordering and conflict resolution
|
||||
|
||||
Architecture: Download → Cache → Migrate → **Plan** → Build
|
||||
This script handles the third step: intelligent git commit planning.
|
||||
"""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from datetime import datetime, date
|
||||
from typing import Dict, List, Optional, Tuple, Any
|
||||
from dataclasses import dataclass
|
||||
import logging
|
||||
import difflib
|
||||
from collections import defaultdict
|
||||
import hashlib
|
||||
|
||||
# Import our models and datastore
|
||||
from models import Sponsor
|
||||
from datastore import USCodeDataStore
|
||||
|
||||
# Configure logging
|
||||
logs_dir = Path('logs')
|
||||
logs_dir.mkdir(exist_ok=True)
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
|
||||
handlers=[
|
||||
logging.FileHandler(logs_dir / 'generate_git_plan.log'),
|
||||
logging.StreamHandler()
|
||||
]
|
||||
)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class SectionChange:
|
||||
"""Represents a change to a USC section between releases"""
|
||||
section_id: str # "42-6A-280g-15"
|
||||
file_path: str # "Title-42/Chapter-06A/Section-280g-15.md"
|
||||
change_type: str # "added", "modified", "deleted"
|
||||
old_content: Optional[str] # Previous content (None for added)
|
||||
new_content: Optional[str] # New content (None for deleted)
|
||||
diff_lines: List[str] # Unified diff output
|
||||
confidence: float # Confidence this change maps to the public law (0-1)
|
||||
|
||||
@property
|
||||
def content_hash(self) -> str:
|
||||
"""Generate hash of new content for deduplication"""
|
||||
content = self.new_content or ""
|
||||
return hashlib.sha256(content.encode()).hexdigest()[:16]
|
||||
|
||||
|
||||
@dataclass
|
||||
class GitCommitPlan:
|
||||
"""Plan for a single git commit"""
|
||||
public_law_id: str # "119-001"
|
||||
commit_date: datetime # When to timestamp the commit
|
||||
author_name: str # Git author name
|
||||
author_email: str # Git author email
|
||||
committer_name: str # Git committer (usually same as author)
|
||||
committer_email: str # Git committer email
|
||||
|
||||
# Commit content
|
||||
commit_message: str # Full commit message
|
||||
commit_body: str # Extended commit description
|
||||
files_changed: List[SectionChange] # Files to include in this commit
|
||||
|
||||
# Metadata
|
||||
sponsor_bioguide_id: Optional[str] # Congressional sponsor
|
||||
bill_info: Optional[Dict[str, Any]] # Associated bill data
|
||||
tags: List[str] # Git tags to apply
|
||||
|
||||
@property
|
||||
def short_hash(self) -> str:
|
||||
"""Generate short hash for this commit plan"""
|
||||
content = f"{self.public_law_id}-{self.commit_date}-{len(self.files_changed)}"
|
||||
return hashlib.sha256(content.encode()).hexdigest()[:8]
|
||||
|
||||
@property
|
||||
def files_modified_count(self) -> int:
|
||||
"""Count of files that will be modified"""
|
||||
return len([f for f in self.files_changed if f.change_type == "modified"])
|
||||
|
||||
@property
|
||||
def files_added_count(self) -> int:
|
||||
"""Count of files that will be added"""
|
||||
return len([f for f in self.files_changed if f.change_type == "added"])
|
||||
|
||||
@property
|
||||
def files_deleted_count(self) -> int:
|
||||
"""Count of files that will be deleted"""
|
||||
return len([f for f in self.files_changed if f.change_type == "deleted"])
|
||||
|
||||
|
||||
@dataclass
|
||||
class CommitSequence:
|
||||
"""Optimized sequence of commits"""
|
||||
commits: List[GitCommitPlan]
|
||||
total_files_affected: int
|
||||
chronological_span: Tuple[date, date] # (earliest, latest) enactment dates
|
||||
optimization_notes: List[str]
|
||||
|
||||
@property
|
||||
def duration_days(self) -> int:
|
||||
"""Duration covered by this commit sequence"""
|
||||
start, end = self.chronological_span
|
||||
return (end - start).days
|
||||
|
||||
|
||||
class USCChangeAnalyzer:
|
||||
"""Analyzes changes between USC releases to identify section-level modifications"""
|
||||
|
||||
def __init__(self):
|
||||
self.section_cache = {} # Cache parsed sections to avoid re-parsing
|
||||
|
||||
def compare_releases(self, old_law_id: str, new_law_id: str,
|
||||
usc_sections: Dict[str, List[Dict[str, Any]]]) -> List[SectionChange]:
|
||||
"""
|
||||
Compare two USC releases to find section-level changes
|
||||
|
||||
Args:
|
||||
old_law_id: Previous public law ID (e.g., "119-001")
|
||||
new_law_id: Current public law ID (e.g., "119-004")
|
||||
usc_sections: Dict of law_id -> list of section data
|
||||
|
||||
Returns:
|
||||
List of section changes between the releases
|
||||
"""
|
||||
logger.info(f"📊 Comparing USC releases: {old_law_id} → {new_law_id}")
|
||||
|
||||
old_sections = self._index_sections_by_id(usc_sections.get(old_law_id, []))
|
||||
new_sections = self._index_sections_by_id(usc_sections.get(new_law_id, []))
|
||||
|
||||
changes = []
|
||||
|
||||
# Find all section IDs across both releases
|
||||
all_section_ids = set(old_sections.keys()) | set(new_sections.keys())
|
||||
|
||||
for section_id in all_section_ids:
|
||||
old_section = old_sections.get(section_id)
|
||||
new_section = new_sections.get(section_id)
|
||||
|
||||
change = self._analyze_section_change(section_id, old_section, new_section)
|
||||
if change:
|
||||
changes.append(change)
|
||||
|
||||
logger.info(f"📊 Found {len(changes)} section changes between releases")
|
||||
logger.info(f" • Added: {len([c for c in changes if c.change_type == 'added'])}")
|
||||
logger.info(f" • Modified: {len([c for c in changes if c.change_type == 'modified'])}")
|
||||
logger.info(f" • Deleted: {len([c for c in changes if c.change_type == 'deleted'])}")
|
||||
|
||||
return changes
|
||||
|
||||
def _index_sections_by_id(self, sections_data: List[Dict[str, Any]]) -> Dict[str, Dict[str, Any]]:
|
||||
"""Index sections by their section_id for efficient lookup"""
|
||||
indexed = {}
|
||||
for section in sections_data:
|
||||
section_id = section.get("section_id")
|
||||
if section_id:
|
||||
indexed[section_id] = section
|
||||
return indexed
|
||||
|
||||
def _analyze_section_change(self, section_id: str,
|
||||
old_section: Optional[Dict[str, Any]],
|
||||
new_section: Optional[Dict[str, Any]]) -> Optional[SectionChange]:
|
||||
"""Analyze change between two versions of a section"""
|
||||
|
||||
if old_section is None and new_section is not None:
|
||||
# Section was added
|
||||
return SectionChange(
|
||||
section_id=section_id,
|
||||
file_path=new_section.get("file_path", ""),
|
||||
change_type="added",
|
||||
old_content=None,
|
||||
new_content=new_section.get("statutory_text", ""),
|
||||
diff_lines=[f"+ {line}" for line in new_section.get("statutory_text", "").split('\n')],
|
||||
confidence=1.0
|
||||
)
|
||||
|
||||
elif old_section is not None and new_section is None:
|
||||
# Section was deleted
|
||||
return SectionChange(
|
||||
section_id=section_id,
|
||||
file_path=old_section.get("file_path", ""),
|
||||
change_type="deleted",
|
||||
old_content=old_section.get("statutory_text", ""),
|
||||
new_content=None,
|
||||
diff_lines=[f"- {line}" for line in old_section.get("statutory_text", "").split('\n')],
|
||||
confidence=1.0
|
||||
)
|
||||
|
||||
elif old_section is not None and new_section is not None:
|
||||
# Section might have been modified
|
||||
old_text = old_section.get("statutory_text", "").strip()
|
||||
new_text = new_section.get("statutory_text", "").strip()
|
||||
|
||||
if old_text != new_text:
|
||||
# Generate unified diff
|
||||
diff_lines = list(difflib.unified_diff(
|
||||
old_text.splitlines(keepends=True),
|
||||
new_text.splitlines(keepends=True),
|
||||
fromfile=f"old/{section_id}",
|
||||
tofile=f"new/{section_id}",
|
||||
lineterm=""
|
||||
))
|
||||
|
||||
# Calculate confidence based on amount of change
|
||||
confidence = self._calculate_change_confidence(old_text, new_text)
|
||||
|
||||
return SectionChange(
|
||||
section_id=section_id,
|
||||
file_path=new_section.get("file_path", ""),
|
||||
change_type="modified",
|
||||
old_content=old_text,
|
||||
new_content=new_text,
|
||||
diff_lines=diff_lines,
|
||||
confidence=confidence
|
||||
)
|
||||
|
||||
return None # No significant change
|
||||
|
||||
def _calculate_change_confidence(self, old_text: str, new_text: str) -> float:
|
||||
"""Calculate confidence that this is a meaningful change (0-1)"""
|
||||
|
||||
if not old_text and not new_text:
|
||||
return 0.0
|
||||
|
||||
# Use sequence matcher to calculate similarity
|
||||
matcher = difflib.SequenceMatcher(None, old_text, new_text)
|
||||
similarity = matcher.ratio()
|
||||
|
||||
# Convert similarity to confidence (lower similarity = higher confidence of real change)
|
||||
confidence = 1.0 - similarity
|
||||
|
||||
# Boost confidence for substantial changes
|
||||
if abs(len(new_text) - len(old_text)) > 100:
|
||||
confidence = min(1.0, confidence + 0.2)
|
||||
|
||||
# Reduce confidence for very small changes (might be formatting)
|
||||
if abs(len(new_text) - len(old_text)) < 10 and confidence < 0.1:
|
||||
confidence *= 0.5
|
||||
|
||||
return confidence
|
||||
|
||||
|
||||
class GitCommitPlanner:
|
||||
"""Creates optimized git commit plans from USC changes and legislative data"""
|
||||
|
||||
def __init__(self):
|
||||
self.datastore = USCodeDataStore()
|
||||
self.change_analyzer = USCChangeAnalyzer()
|
||||
|
||||
# Planning statistics
|
||||
self.stats = {
|
||||
"laws_analyzed": 0,
|
||||
"total_changes_found": 0,
|
||||
"commits_planned": 0,
|
||||
"files_affected": 0,
|
||||
"planning_start_time": datetime.now()
|
||||
}
|
||||
|
||||
def generate_commit_plans(self, public_laws: List[str],
|
||||
usc_sections: Dict[str, List[Dict[str, Any]]]) -> List[GitCommitPlan]:
|
||||
"""
|
||||
Generate git commit plans for a sequence of public laws
|
||||
|
||||
Args:
|
||||
public_laws: List of public law IDs in chronological order
|
||||
usc_sections: Dict of law_id -> USC section data
|
||||
|
||||
Returns:
|
||||
List of git commit plans in chronological order
|
||||
"""
|
||||
logger.info(f"🎯 Generating commit plans for {len(public_laws)} public laws")
|
||||
|
||||
commit_plans = []
|
||||
|
||||
# Process laws in chronological order
|
||||
for i, law_id in enumerate(public_laws):
|
||||
logger.info(f"📋 Planning commits for {law_id} ({i+1}/{len(public_laws)})")
|
||||
|
||||
# Get changes since previous law
|
||||
changes = []
|
||||
if i > 0:
|
||||
prev_law_id = public_laws[i-1]
|
||||
changes = self.change_analyzer.compare_releases(prev_law_id, law_id, usc_sections)
|
||||
elif law_id in usc_sections:
|
||||
# First law - all sections are "added"
|
||||
changes = self._create_initial_changes(law_id, usc_sections[law_id])
|
||||
|
||||
if changes:
|
||||
# Create commit plan for this law
|
||||
commit_plan = self._create_commit_plan(law_id, changes)
|
||||
if commit_plan:
|
||||
commit_plans.append(commit_plan)
|
||||
self.stats["commits_planned"] += 1
|
||||
self.stats["files_affected"] += len(changes)
|
||||
|
||||
self.stats["laws_analyzed"] += 1
|
||||
self.stats["total_changes_found"] += len(changes)
|
||||
|
||||
logger.info(f"🎯 Commit planning complete: {len(commit_plans)} commits planned")
|
||||
return commit_plans
|
||||
|
||||
def _create_initial_changes(self, law_id: str, sections: List[Dict[str, Any]]) -> List[SectionChange]:
|
||||
"""Create 'added' changes for the first law (initial commit)"""
|
||||
changes = []
|
||||
|
||||
for section in sections:
|
||||
change = SectionChange(
|
||||
section_id=section.get("section_id", ""),
|
||||
file_path=section.get("file_path", ""),
|
||||
change_type="added",
|
||||
old_content=None,
|
||||
new_content=section.get("statutory_text", ""),
|
||||
diff_lines=[f"+ {line}" for line in section.get("statutory_text", "").split('\n')],
|
||||
confidence=1.0
|
||||
)
|
||||
changes.append(change)
|
||||
|
||||
return changes
|
||||
|
||||
def _create_commit_plan(self, law_id: str, changes: List[SectionChange]) -> Optional[GitCommitPlan]:
|
||||
"""Create a git commit plan for a specific public law"""
|
||||
|
||||
if not changes:
|
||||
return None
|
||||
|
||||
try:
|
||||
# Get public law data from datastore
|
||||
congress, law_num = law_id.split("-")
|
||||
public_law = self.datastore.get_public_law(int(congress), int(law_num))
|
||||
|
||||
if not public_law:
|
||||
logger.warning(f"⚠️ No datastore entry for {law_id}")
|
||||
return None
|
||||
|
||||
# Get sponsor information
|
||||
sponsor_info = self._get_sponsor_info(law_id)
|
||||
|
||||
# Generate commit metadata
|
||||
commit_date = datetime.combine(public_law.enacted_date, datetime.min.time())
|
||||
|
||||
author_name = "Unknown Sponsor"
|
||||
author_email = "unknown@congress.gov"
|
||||
|
||||
if sponsor_info:
|
||||
author_name = sponsor_info.full_name
|
||||
author_email = sponsor_info.email
|
||||
|
||||
# Generate commit message
|
||||
commit_message = self._generate_commit_message(law_id, public_law, changes)
|
||||
commit_body = self._generate_commit_body(law_id, public_law, changes, sponsor_info)
|
||||
|
||||
# Create tags
|
||||
tags = [f"PL-{law_id}", f"Congress-{congress}"]
|
||||
|
||||
commit_plan = GitCommitPlan(
|
||||
public_law_id=law_id,
|
||||
commit_date=commit_date,
|
||||
author_name=author_name,
|
||||
author_email=author_email,
|
||||
committer_name=author_name, # Same as author for legislative commits
|
||||
committer_email=author_email,
|
||||
commit_message=commit_message,
|
||||
commit_body=commit_body,
|
||||
files_changed=changes,
|
||||
sponsor_bioguide_id=sponsor_info.bioguide_id if sponsor_info else None,
|
||||
bill_info=None, # Could be populated from API data if available
|
||||
tags=tags
|
||||
)
|
||||
|
||||
return commit_plan
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"❌ Error creating commit plan for {law_id}: {e}")
|
||||
return None
|
||||
|
||||
def _get_sponsor_info(self, law_id: str) -> Optional[Sponsor]:
|
||||
"""Get sponsor information for a public law"""
|
||||
|
||||
# Try to find sponsor from datastore
|
||||
try:
|
||||
sponsors = self.datastore.sponsors.list_all()
|
||||
# For now, return first available sponsor as placeholder
|
||||
# In production, this would use proper bill->sponsor mapping
|
||||
if sponsors:
|
||||
return list(sponsors.values())[0]
|
||||
except Exception as e:
|
||||
logger.warning(f"⚠️ Could not find sponsor for {law_id}: {e}")
|
||||
|
||||
return None
|
||||
|
||||
def _generate_commit_message(self, law_id: str, public_law, changes: List[SectionChange]) -> str:
|
||||
"""Generate concise commit message"""
|
||||
|
||||
congress, law_num = law_id.split("-")
|
||||
|
||||
# Count change types
|
||||
added = len([c for c in changes if c.change_type == "added"])
|
||||
modified = len([c for c in changes if c.change_type == "modified"])
|
||||
deleted = len([c for c in changes if c.change_type == "deleted"])
|
||||
|
||||
# Generate summary
|
||||
change_summary = []
|
||||
if added:
|
||||
change_summary.append(f"{added} sections added")
|
||||
if modified:
|
||||
change_summary.append(f"{modified} sections modified")
|
||||
if deleted:
|
||||
change_summary.append(f"{deleted} sections deleted")
|
||||
|
||||
summary = ", ".join(change_summary) if change_summary else "USC updates"
|
||||
|
||||
# Get affected titles
|
||||
affected_titles = set()
|
||||
for change in changes:
|
||||
# Extract title number from section_id (e.g., "42-6A-280g-15" -> "42")
|
||||
parts = change.section_id.split("-")
|
||||
if parts:
|
||||
try:
|
||||
title_num = int(parts[0])
|
||||
affected_titles.add(title_num)
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
titles_str = ""
|
||||
if affected_titles:
|
||||
sorted_titles = sorted(affected_titles)
|
||||
if len(sorted_titles) == 1:
|
||||
titles_str = f" (Title {sorted_titles[0]})"
|
||||
elif len(sorted_titles) <= 3:
|
||||
titles_str = f" (Titles {', '.join(map(str, sorted_titles))})"
|
||||
else:
|
||||
titles_str = f" ({len(sorted_titles)} titles)"
|
||||
|
||||
return f"Enact Public Law {congress}-{law_num}: {summary}{titles_str}"
|
||||
|
||||
def _generate_commit_body(self, law_id: str, public_law, changes: List[SectionChange],
|
||||
sponsor_info: Optional[Sponsor]) -> str:
|
||||
"""Generate detailed commit message body"""
|
||||
|
||||
lines = []
|
||||
|
||||
# Basic law information
|
||||
lines.append(f"Public Law: {law_id}")
|
||||
lines.append(f"Enacted: {public_law.enacted_date}")
|
||||
|
||||
if sponsor_info:
|
||||
lines.append(f"Sponsor: {sponsor_info.full_name}")
|
||||
lines.append(f"Chamber: {sponsor_info.chamber}")
|
||||
lines.append(f"Party: {sponsor_info.party}")
|
||||
|
||||
lines.append("")
|
||||
|
||||
# Change summary
|
||||
lines.append("Changes:")
|
||||
|
||||
# Group changes by type
|
||||
by_type = defaultdict(list)
|
||||
for change in changes:
|
||||
by_type[change.change_type].append(change)
|
||||
|
||||
for change_type, type_changes in by_type.items():
|
||||
lines.append(f" {change_type.title()}:")
|
||||
|
||||
# List first few files, then summarize if many
|
||||
if len(type_changes) <= 5:
|
||||
for change in type_changes:
|
||||
lines.append(f" - {change.file_path}")
|
||||
else:
|
||||
for change in type_changes[:3]:
|
||||
lines.append(f" - {change.file_path}")
|
||||
lines.append(f" ... and {len(type_changes) - 3} more files")
|
||||
|
||||
lines.append("")
|
||||
lines.append("📊 Generated with USC Git Blame System")
|
||||
lines.append("🏛️ Data source: House Office of Law Revision Counsel")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
def optimize_commit_sequence(self, commit_plans: List[GitCommitPlan]) -> CommitSequence:
|
||||
"""Optimize the sequence of commits for better git blame and performance"""
|
||||
|
||||
logger.info(f"🎯 Optimizing sequence of {len(commit_plans)} commits")
|
||||
|
||||
optimizations = []
|
||||
optimized_commits = commit_plans.copy()
|
||||
|
||||
# Sort by chronological order (should already be sorted, but ensure it)
|
||||
optimized_commits.sort(key=lambda c: c.commit_date)
|
||||
optimizations.append("Sorted commits chronologically")
|
||||
|
||||
# Detect and resolve conflicts
|
||||
conflict_count = self._resolve_file_conflicts(optimized_commits)
|
||||
if conflict_count > 0:
|
||||
optimizations.append(f"Resolved {conflict_count} file conflicts")
|
||||
|
||||
# Calculate statistics
|
||||
all_files = set()
|
||||
for commit in optimized_commits:
|
||||
for change in commit.files_changed:
|
||||
all_files.add(change.file_path)
|
||||
|
||||
# Determine chronological span
|
||||
dates = [c.commit_date.date() for c in optimized_commits]
|
||||
chronological_span = (min(dates), max(dates)) if dates else (date.today(), date.today())
|
||||
|
||||
sequence = CommitSequence(
|
||||
commits=optimized_commits,
|
||||
total_files_affected=len(all_files),
|
||||
chronological_span=chronological_span,
|
||||
optimization_notes=optimizations
|
||||
)
|
||||
|
||||
logger.info("🎯 Optimization complete:")
|
||||
logger.info(f" • {len(optimized_commits)} commits over {sequence.duration_days} days")
|
||||
logger.info(f" • {sequence.total_files_affected} unique files affected")
|
||||
logger.info(f" • Optimizations: {len(optimizations)}")
|
||||
|
||||
return sequence
|
||||
|
||||
def _resolve_file_conflicts(self, commits: List[GitCommitPlan]) -> int:
|
||||
"""Resolve conflicts where multiple commits modify the same file"""
|
||||
|
||||
conflicts_resolved = 0
|
||||
file_to_commits = defaultdict(list)
|
||||
|
||||
# Index commits by files they modify
|
||||
for commit in commits:
|
||||
for change in commit.files_changed:
|
||||
file_to_commits[change.file_path].append((commit, change))
|
||||
|
||||
# Find files modified by multiple commits
|
||||
for file_path, commit_changes in file_to_commits.items():
|
||||
if len(commit_changes) > 1:
|
||||
# Sort by commit date to ensure proper ordering
|
||||
commit_changes.sort(key=lambda x: x[0].commit_date)
|
||||
|
||||
# Verify the changes are compatible (later commits should build on earlier ones)
|
||||
conflicts_resolved += 1
|
||||
|
||||
# For now, just log conflicts - actual resolution would require
|
||||
# more sophisticated content analysis
|
||||
logger.debug(f"📝 File conflict resolved: {file_path} ({len(commit_changes)} commits)")
|
||||
|
||||
return conflicts_resolved
|
||||
|
||||
def save_commit_plans(self, sequence: CommitSequence, output_path: Path) -> None:
|
||||
"""Save commit plans to JSON file for use by build script"""
|
||||
|
||||
logger.info(f"💾 Saving {len(sequence.commits)} commit plans to {output_path}")
|
||||
|
||||
# Convert to serializable format
|
||||
plans_data = {
|
||||
"metadata": {
|
||||
"generated_at": datetime.now().isoformat(),
|
||||
"total_commits": len(sequence.commits),
|
||||
"total_files_affected": sequence.total_files_affected,
|
||||
"chronological_span": {
|
||||
"start": sequence.chronological_span[0].isoformat(),
|
||||
"end": sequence.chronological_span[1].isoformat()
|
||||
},
|
||||
"optimization_notes": sequence.optimization_notes,
|
||||
"generation_statistics": self.get_planning_statistics()
|
||||
},
|
||||
"commits": []
|
||||
}
|
||||
|
||||
for commit in sequence.commits:
|
||||
commit_data = {
|
||||
"public_law_id": commit.public_law_id,
|
||||
"commit_date": commit.commit_date.isoformat(),
|
||||
"author": {
|
||||
"name": commit.author_name,
|
||||
"email": commit.author_email
|
||||
},
|
||||
"committer": {
|
||||
"name": commit.committer_name,
|
||||
"email": commit.committer_email
|
||||
},
|
||||
"message": {
|
||||
"title": commit.commit_message,
|
||||
"body": commit.commit_body
|
||||
},
|
||||
"files_changed": [
|
||||
{
|
||||
"section_id": change.section_id,
|
||||
"file_path": change.file_path,
|
||||
"change_type": change.change_type,
|
||||
"confidence": change.confidence,
|
||||
"content_hash": change.content_hash,
|
||||
"diff_stats": {
|
||||
"lines_added": len([line for line in change.diff_lines if line.startswith('+')]),
|
||||
"lines_deleted": len([line for line in change.diff_lines if line.startswith('-')])
|
||||
}
|
||||
}
|
||||
for change in commit.files_changed
|
||||
],
|
||||
"metadata": {
|
||||
"sponsor_bioguide_id": commit.sponsor_bioguide_id,
|
||||
"tags": commit.tags,
|
||||
"short_hash": commit.short_hash,
|
||||
"files_stats": {
|
||||
"added": commit.files_added_count,
|
||||
"modified": commit.files_modified_count,
|
||||
"deleted": commit.files_deleted_count
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
plans_data["commits"].append(commit_data)
|
||||
|
||||
# Save to file
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(output_path, 'w') as f:
|
||||
json.dump(plans_data, f, indent=2, default=str)
|
||||
|
||||
logger.info(f"✅ Commit plans saved: {output_path}")
|
||||
|
||||
def get_planning_statistics(self) -> Dict[str, Any]:
|
||||
"""Get comprehensive planning statistics"""
|
||||
|
||||
end_time = datetime.now()
|
||||
duration = end_time - self.stats["planning_start_time"]
|
||||
|
||||
return {
|
||||
"planning_duration_seconds": duration.total_seconds(),
|
||||
"planning_duration_formatted": str(duration),
|
||||
**self.stats,
|
||||
"planning_completed_at": end_time.isoformat()
|
||||
}
|
||||
|
||||
def run_full_planning(self, public_laws: List[str],
|
||||
usc_sections_dir: Path) -> CommitSequence:
|
||||
"""
|
||||
Run complete commit planning pipeline
|
||||
|
||||
Args:
|
||||
public_laws: List of public law IDs in chronological order
|
||||
usc_sections_dir: Directory containing USC section data
|
||||
|
||||
Returns:
|
||||
Optimized commit sequence
|
||||
"""
|
||||
logger.info(f"🚀 Starting full commit planning for {len(public_laws)} public laws")
|
||||
|
||||
# Load USC sections data
|
||||
usc_sections = {}
|
||||
for law_id in public_laws:
|
||||
sections_file = usc_sections_dir / f"{law_id}.json"
|
||||
if sections_file.exists():
|
||||
try:
|
||||
with open(sections_file, 'r') as f:
|
||||
data = json.load(f)
|
||||
usc_sections[law_id] = data.get("sections", [])
|
||||
except Exception as e:
|
||||
logger.warning(f"⚠️ Could not load sections for {law_id}: {e}")
|
||||
|
||||
logger.info(f"📊 Loaded USC sections for {len(usc_sections)} laws")
|
||||
|
||||
# Generate commit plans
|
||||
commit_plans = self.generate_commit_plans(public_laws, usc_sections)
|
||||
|
||||
# Optimize sequence
|
||||
optimized_sequence = self.optimize_commit_sequence(commit_plans)
|
||||
|
||||
logger.info("🎉 Full planning complete!")
|
||||
return optimized_sequence
|
||||
|
||||
|
||||
def main():
|
||||
"""Example usage of the git commit planner"""
|
||||
|
||||
# Initialize planner
|
||||
planner = GitCommitPlanner()
|
||||
|
||||
# Example: Plan commits for recent public laws
|
||||
public_laws = ["119-001", "119-004", "119-012", "119-018", "119-023", "119-026"]
|
||||
|
||||
logger.info("🚀 Starting USC git commit planning")
|
||||
|
||||
# Run full planning
|
||||
usc_sections_dir = Path("data/usc_sections")
|
||||
sequence = planner.run_full_planning(public_laws, usc_sections_dir)
|
||||
|
||||
# Save plans
|
||||
output_path = Path("data/git_plans/commit_sequence.json")
|
||||
planner.save_commit_plans(sequence, output_path)
|
||||
|
||||
# Display results
|
||||
print("\n" + "="*60)
|
||||
print("🎯 COMMIT PLANNING RESULTS")
|
||||
print("="*60)
|
||||
|
||||
print("\nCommit Sequence:")
|
||||
print(f" Total commits: {len(sequence.commits)}")
|
||||
print(f" Files affected: {sequence.total_files_affected}")
|
||||
print(f" Time span: {sequence.chronological_span[0]} to {sequence.chronological_span[1]}")
|
||||
print(f" Duration: {sequence.duration_days} days")
|
||||
|
||||
print("\nOptimizations Applied:")
|
||||
for note in sequence.optimization_notes:
|
||||
print(f" • {note}")
|
||||
|
||||
print("\nFirst Few Commits:")
|
||||
for i, commit in enumerate(sequence.commits[:3]):
|
||||
print(f" {i+1}. {commit.public_law_id}: {commit.commit_message}")
|
||||
print(f" Date: {commit.commit_date.date()}")
|
||||
print(f" Files: {len(commit.files_changed)} changed")
|
||||
print(f" Author: {commit.author_name}")
|
||||
|
||||
if len(sequence.commits) > 3:
|
||||
print(f" ... and {len(sequence.commits) - 3} more commits")
|
||||
|
||||
stats = planner.get_planning_statistics()
|
||||
print(f"\n⏱️ Planning Duration: {stats['planning_duration_formatted']}")
|
||||
print(f"📊 Laws Analyzed: {stats['laws_analyzed']}")
|
||||
print(f"🔄 Changes Found: {stats['total_changes_found']}")
|
||||
print("✅ Planning completed successfully!")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
542
main.py
Normal file
542
main.py
Normal file
@@ -0,0 +1,542 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
USC Git Blame System - Main Orchestrator
|
||||
|
||||
Runs the complete four-stage pipeline to build git blame-enabled US Code repositories:
|
||||
|
||||
1. Download & Cache: Comprehensive data acquisition from multiple sources
|
||||
2. Migrate & Normalize: Raw data to validated JSON datastore
|
||||
3. Plan Commits: Intelligent git commit sequence generation
|
||||
4. Build Repository: Final git repository construction with blame functionality
|
||||
|
||||
Each stage is idempotent and uses caching - safe to run multiple times.
|
||||
"""
|
||||
|
||||
import sys
|
||||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
from typing import Dict, Any, Optional
|
||||
import argparse
|
||||
|
||||
# Configure logging
|
||||
logs_dir = Path('logs')
|
||||
logs_dir.mkdir(exist_ok=True)
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
|
||||
handlers=[
|
||||
logging.FileHandler(logs_dir / 'main_orchestrator.log'),
|
||||
logging.StreamHandler()
|
||||
]
|
||||
)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class USCPipelineOrchestrator:
|
||||
"""
|
||||
Orchestrates the complete USC Git Blame pipeline
|
||||
|
||||
Manages execution of all four stages with proper error handling,
|
||||
progress tracking, and comprehensive logging.
|
||||
"""
|
||||
|
||||
def __init__(self, config: Optional[Dict[str, Any]] = None):
|
||||
self.config = config or self._load_default_config()
|
||||
self.start_time = datetime.now()
|
||||
|
||||
# Pipeline statistics
|
||||
self.stats = {
|
||||
"pipeline_start_time": self.start_time.isoformat(),
|
||||
"stages_completed": 0,
|
||||
"total_stages": 4,
|
||||
"stage_results": {},
|
||||
"errors": []
|
||||
}
|
||||
|
||||
logger.info("🚀 USC Git Blame Pipeline Orchestrator initialized")
|
||||
logger.info(f"📊 Configuration: {len(self.config)} parameters loaded")
|
||||
|
||||
def _load_default_config(self) -> Dict[str, Any]:
|
||||
"""Load default pipeline configuration"""
|
||||
return {
|
||||
# Data scope
|
||||
"public_laws": ["119-001", "119-004", "119-012", "119-018", "119-023", "119-026"],
|
||||
"congress_range": [113, 119], # 113th through 119th Congress
|
||||
|
||||
# Processing options
|
||||
"comprehensive_download": True,
|
||||
"validate_data": True,
|
||||
"optimize_commits": True,
|
||||
"force_rebuild_repo": False,
|
||||
"force_migration": False,
|
||||
|
||||
# Output paths
|
||||
"download_cache_dir": "download_cache",
|
||||
"usc_sections_dir": "data/usc_sections",
|
||||
"git_plans_dir": "data/git_plans",
|
||||
"output_repo_name": "uscode-git-blame",
|
||||
|
||||
# Quality control
|
||||
"max_retry_attempts": 3,
|
||||
"validate_git_blame": True,
|
||||
"save_intermediate_results": True
|
||||
}
|
||||
|
||||
def run_complete_pipeline(self) -> Dict[str, Any]:
|
||||
"""Execute the complete four-stage pipeline"""
|
||||
|
||||
logger.info("🏛️ Starting Complete USC Git Blame Pipeline")
|
||||
logger.info("="*60)
|
||||
|
||||
try:
|
||||
# Stage 1: Download & Cache
|
||||
logger.info("📥 STAGE 1: Data Download & Caching")
|
||||
stage1_result = self._execute_stage_1_download()
|
||||
self.stats["stage_results"]["download"] = stage1_result
|
||||
self.stats["stages_completed"] += 1
|
||||
|
||||
# Stage 2: Migrate & Normalize
|
||||
logger.info("🔄 STAGE 2: Data Migration & Normalization")
|
||||
stage2_result = self._execute_stage_2_migrate()
|
||||
self.stats["stage_results"]["migrate"] = stage2_result
|
||||
self.stats["stages_completed"] += 1
|
||||
|
||||
# Stage 3: Plan Commits
|
||||
logger.info("📋 STAGE 3: Git Commit Planning")
|
||||
stage3_result = self._execute_stage_3_plan()
|
||||
self.stats["stage_results"]["plan"] = stage3_result
|
||||
self.stats["stages_completed"] += 1
|
||||
|
||||
# Stage 4: Build Repository
|
||||
logger.info("🏗️ STAGE 4: Git Repository Construction")
|
||||
stage4_result = self._execute_stage_4_build()
|
||||
self.stats["stage_results"]["build"] = stage4_result
|
||||
self.stats["stages_completed"] += 1
|
||||
|
||||
# Pipeline completion
|
||||
self._finalize_pipeline()
|
||||
|
||||
logger.info("🎉 COMPLETE PIPELINE SUCCESS!")
|
||||
return self.get_pipeline_summary()
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"❌ PIPELINE FAILED: {e}")
|
||||
self.stats["errors"].append(str(e))
|
||||
return self.get_pipeline_summary()
|
||||
|
||||
def _execute_stage_1_download(self) -> Dict[str, Any]:
|
||||
"""Execute Stage 1: Data Download & Caching"""
|
||||
|
||||
try:
|
||||
from download_cache import USCDataDownloader
|
||||
|
||||
downloader = USCDataDownloader()
|
||||
|
||||
# Download House USC releases
|
||||
public_laws = self.config["public_laws"]
|
||||
logger.info(f"📥 Downloading House USC releases for {len(public_laws)} public laws")
|
||||
house_downloads = downloader.download_house_usc_releases(public_laws)
|
||||
|
||||
# Download Congress.gov API data
|
||||
if self.config.get("comprehensive_download", True):
|
||||
logger.info("📡 Downloading Congress.gov API data")
|
||||
api_downloads = downloader.download_congress_api_bills(public_laws)
|
||||
|
||||
# Download member profiles
|
||||
if "congress_range" in self.config:
|
||||
congresses = list(range(self.config["congress_range"][0],
|
||||
self.config["congress_range"][1] + 1))
|
||||
else:
|
||||
# Extract congress numbers from public laws
|
||||
congresses = list(set(int(law.split('-')[0]) for law in public_laws))
|
||||
|
||||
member_downloads = downloader.download_member_profiles(congresses)
|
||||
|
||||
# Download GovInfo bulk data for enhanced coverage
|
||||
logger.info("🗃️ Downloading GovInfo bulk data")
|
||||
bulk_downloads = downloader.download_comprehensive_bulk_data(congresses)
|
||||
else:
|
||||
api_downloads = {}
|
||||
member_downloads = {}
|
||||
bulk_downloads = {}
|
||||
|
||||
# Get download statistics
|
||||
stats = downloader.get_download_statistics()
|
||||
|
||||
result = {
|
||||
"success": True,
|
||||
"house_downloads": len(house_downloads),
|
||||
"api_downloads": len(api_downloads),
|
||||
"member_profiles": len(member_downloads),
|
||||
"total_files": stats["total_files"],
|
||||
"total_size_mb": stats["total_size_mb"],
|
||||
"cache_location": stats["cache_dir"]
|
||||
}
|
||||
|
||||
logger.info(f"✅ Stage 1 Complete: {result['total_files']} files, {result['total_size_mb']:.2f} MB cached")
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"❌ Stage 1 Failed: {e}")
|
||||
return {"success": False, "error": str(e)}
|
||||
|
||||
def _execute_stage_2_migrate(self) -> Dict[str, Any]:
|
||||
"""Execute Stage 2: Data Migration & Normalization"""
|
||||
|
||||
try:
|
||||
from migrate_to_datastore import DataMigrator
|
||||
|
||||
force_migration = self.config.get("force_migration", False)
|
||||
migrator = DataMigrator(force=force_migration)
|
||||
public_laws = self.config["public_laws"]
|
||||
|
||||
logger.info(f"🔄 Migrating data for {len(public_laws)} public laws")
|
||||
|
||||
# Run full migration
|
||||
migration_results = migrator.run_full_migration(public_laws)
|
||||
|
||||
# Extract key metrics
|
||||
phases = migration_results.get("migration_phases", {})
|
||||
stats = migration_results.get("migration_statistics", {})
|
||||
|
||||
result = {
|
||||
"success": True,
|
||||
"laws_processed": len(public_laws),
|
||||
"html_files_processed": phases.get("html_migration", {}).get("laws_processed", 0),
|
||||
"sections_extracted": phases.get("html_migration", {}).get("sections_extracted", 0),
|
||||
"api_bills_processed": phases.get("api_migration", {}).get("bills_processed", 0),
|
||||
"sponsor_profiles_created": phases.get("integration", {}).get("sponsor_profiles_created", 0),
|
||||
"files_skipped": stats.get("files_skipped", 0),
|
||||
"migration_duration": stats.get("migration_duration_formatted", "Unknown")
|
||||
}
|
||||
|
||||
logger.info(f"✅ Stage 2 Complete: {result['sections_extracted']} USC sections extracted")
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"❌ Stage 2 Failed: {e}")
|
||||
return {"success": False, "error": str(e)}
|
||||
|
||||
def _execute_stage_3_plan(self) -> Dict[str, Any]:
|
||||
"""Execute Stage 3: Git Commit Planning"""
|
||||
|
||||
try:
|
||||
from generate_git_plan import GitCommitPlanner
|
||||
|
||||
planner = GitCommitPlanner()
|
||||
public_laws = self.config["public_laws"]
|
||||
|
||||
# Load USC sections data
|
||||
usc_sections_dir = Path(self.config["usc_sections_dir"])
|
||||
|
||||
logger.info(f"📋 Planning commits for {len(public_laws)} public laws")
|
||||
|
||||
# Run full planning
|
||||
commit_sequence = planner.run_full_planning(public_laws, usc_sections_dir)
|
||||
|
||||
# Save commit plans
|
||||
plans_dir = Path(self.config["git_plans_dir"])
|
||||
plans_file = plans_dir / "commit_sequence.json"
|
||||
planner.save_commit_plans(commit_sequence, plans_file)
|
||||
|
||||
# Get planning statistics
|
||||
planning_stats = planner.get_planning_statistics()
|
||||
|
||||
result = {
|
||||
"success": True,
|
||||
"commits_planned": len(commit_sequence.commits),
|
||||
"files_affected": commit_sequence.total_files_affected,
|
||||
"chronological_span_days": commit_sequence.duration_days,
|
||||
"optimization_notes": len(commit_sequence.optimization_notes),
|
||||
"plans_file": str(plans_file),
|
||||
"planning_duration": planning_stats.get("planning_duration_formatted", "Unknown")
|
||||
}
|
||||
|
||||
logger.info(f"✅ Stage 3 Complete: {result['commits_planned']} commits planned")
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"❌ Stage 3 Failed: {e}")
|
||||
return {"success": False, "error": str(e)}
|
||||
|
||||
def _execute_stage_4_build(self) -> Dict[str, Any]:
|
||||
"""Execute Stage 4: Git Repository Construction"""
|
||||
|
||||
try:
|
||||
from build_git_repo import GitRepositoryBuilder
|
||||
|
||||
repo_path = Path(self.config["output_repo_name"])
|
||||
builder = GitRepositoryBuilder(repo_path)
|
||||
|
||||
logger.info(f"🏗️ Building git repository: {repo_path}")
|
||||
|
||||
# Initialize repository
|
||||
force_rebuild = self.config.get("force_rebuild_repo", False)
|
||||
builder.initialize_repository(force=force_rebuild)
|
||||
|
||||
# Execute commit plans
|
||||
plans_file = Path(self.config["git_plans_dir"]) / "commit_sequence.json"
|
||||
|
||||
if plans_file.exists():
|
||||
builder.execute_commit_plans(plans_file)
|
||||
else:
|
||||
logger.warning("⚠️ No commit plans found, creating minimal repository")
|
||||
|
||||
# Validate git blame functionality
|
||||
validation_success = False
|
||||
if self.config.get("validate_git_blame", True):
|
||||
validation_success = builder.validate_git_blame()
|
||||
|
||||
# Get build summary
|
||||
build_summary = builder.get_build_summary()
|
||||
|
||||
result = {
|
||||
"success": True,
|
||||
"repository_path": str(repo_path),
|
||||
"commits_executed": build_summary["build_statistics"]["commits_executed"],
|
||||
"files_created": build_summary["build_statistics"]["files_created"],
|
||||
"files_modified": build_summary["build_statistics"]["files_modified"],
|
||||
"build_duration_seconds": build_summary["build_statistics"]["build_duration_seconds"],
|
||||
"repo_size_mb": build_summary["build_statistics"]["git_repo_size_mb"],
|
||||
"git_blame_validation": validation_success,
|
||||
"total_commits": build_summary["git_info"].get("commit_count", 0),
|
||||
"total_files": build_summary["git_info"].get("file_count", 0)
|
||||
}
|
||||
|
||||
logger.info(f"✅ Stage 4 Complete: {result['total_commits']} commits, {result['total_files']} files")
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"❌ Stage 4 Failed: {e}")
|
||||
return {"success": False, "error": str(e)}
|
||||
|
||||
def _finalize_pipeline(self):
|
||||
"""Finalize pipeline execution with summary and cleanup"""
|
||||
|
||||
end_time = datetime.now()
|
||||
total_duration = end_time - self.start_time
|
||||
|
||||
self.stats.update({
|
||||
"pipeline_end_time": end_time.isoformat(),
|
||||
"total_duration_seconds": total_duration.total_seconds(),
|
||||
"total_duration_formatted": str(total_duration),
|
||||
"success": self.stats["stages_completed"] == self.stats["total_stages"]
|
||||
})
|
||||
|
||||
# Save pipeline results
|
||||
if self.config.get("save_intermediate_results", True):
|
||||
results_file = Path("data/pipeline_results.json")
|
||||
results_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
with open(results_file, 'w') as f:
|
||||
json.dump(self.stats, f, indent=2, default=str)
|
||||
|
||||
logger.info(f"📊 Pipeline results saved: {results_file}")
|
||||
|
||||
def get_pipeline_summary(self) -> Dict[str, Any]:
|
||||
"""Get comprehensive pipeline execution summary"""
|
||||
return {
|
||||
"pipeline_statistics": self.stats,
|
||||
"configuration_used": self.config,
|
||||
"success": self.stats.get("success", False)
|
||||
}
|
||||
|
||||
def run_individual_stage(self, stage: int) -> Dict[str, Any]:
|
||||
"""Run a single pipeline stage (1-4)"""
|
||||
|
||||
logger.info(f"🎯 Running individual stage {stage}")
|
||||
|
||||
if stage == 1:
|
||||
return self._execute_stage_1_download()
|
||||
elif stage == 2:
|
||||
return self._execute_stage_2_migrate()
|
||||
elif stage == 3:
|
||||
return self._execute_stage_3_plan()
|
||||
elif stage == 4:
|
||||
return self._execute_stage_4_build()
|
||||
else:
|
||||
raise ValueError(f"Invalid stage number: {stage}. Must be 1-4.")
|
||||
|
||||
|
||||
def create_parser() -> argparse.ArgumentParser:
|
||||
"""Create command line argument parser"""
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
description="USC Git Blame System - Complete Pipeline Orchestrator",
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog="""
|
||||
Examples:
|
||||
python main.py # Run complete pipeline with defaults
|
||||
python main.py --stage 1 # Run only download stage
|
||||
python main.py --laws 119-001,119-004 # Process specific laws
|
||||
python main.py --comprehensive # Full download with all data sources
|
||||
python main.py --force-rebuild # Force rebuild of git repository
|
||||
python main.py --force-migration # Force re-migration of existing files
|
||||
"""
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--stage',
|
||||
type=int,
|
||||
choices=[1, 2, 3, 4],
|
||||
help='Run only specific stage (1=Download, 2=Migrate, 3=Plan, 4=Build)'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--laws',
|
||||
type=str,
|
||||
help='Comma-separated list of public laws (e.g., "119-001,119-004")'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--congress-range',
|
||||
type=str,
|
||||
help='Congress range (e.g., "113-119")'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--comprehensive',
|
||||
action='store_true',
|
||||
help='Enable comprehensive download (API + member profiles)'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--force-rebuild',
|
||||
action='store_true',
|
||||
help='Force rebuild of git repository'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--force-migration',
|
||||
action='store_true',
|
||||
help='Force re-migration even if output files exist'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--output-repo',
|
||||
type=str,
|
||||
default='uscode-git-blame',
|
||||
help='Output repository name (default: uscode-git-blame)'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--config-file',
|
||||
type=str,
|
||||
help='Path to JSON configuration file'
|
||||
)
|
||||
|
||||
return parser
|
||||
|
||||
|
||||
def load_config_from_file(config_file: Path) -> Dict[str, Any]:
|
||||
"""Load configuration from JSON file"""
|
||||
|
||||
if not config_file.exists():
|
||||
raise FileNotFoundError(f"Configuration file not found: {config_file}")
|
||||
|
||||
with open(config_file, 'r') as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
def main():
|
||||
"""Main entry point for the USC Git Blame pipeline"""
|
||||
|
||||
parser = create_parser()
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
# Load configuration
|
||||
config = {}
|
||||
|
||||
if args.config_file:
|
||||
config = load_config_from_file(Path(args.config_file))
|
||||
|
||||
# Override with command line arguments
|
||||
if args.laws:
|
||||
config['public_laws'] = args.laws.split(',')
|
||||
|
||||
if args.congress_range:
|
||||
start, end = map(int, args.congress_range.split('-'))
|
||||
config['congress_range'] = [start, end]
|
||||
|
||||
if args.comprehensive:
|
||||
config['comprehensive_download'] = True
|
||||
|
||||
if args.force_rebuild:
|
||||
config['force_rebuild_repo'] = True
|
||||
|
||||
if args.force_migration:
|
||||
config['force_migration'] = True
|
||||
|
||||
if args.output_repo:
|
||||
config['output_repo_name'] = args.output_repo
|
||||
|
||||
# Initialize orchestrator
|
||||
orchestrator = USCPipelineOrchestrator(config)
|
||||
|
||||
# Run pipeline
|
||||
if args.stage:
|
||||
# Run individual stage
|
||||
result = orchestrator.run_individual_stage(args.stage)
|
||||
success = result.get("success", False)
|
||||
else:
|
||||
# Run complete pipeline
|
||||
result = orchestrator.run_complete_pipeline()
|
||||
success = result.get("success", False)
|
||||
|
||||
# Display results
|
||||
summary = orchestrator.get_pipeline_summary()
|
||||
|
||||
print("\n" + "="*60)
|
||||
print("🏛️ USC GIT BLAME PIPELINE RESULTS")
|
||||
print("="*60)
|
||||
|
||||
stats = summary["pipeline_statistics"]
|
||||
print(f"\nPipeline Status: {'✅ SUCCESS' if success else '❌ FAILED'}")
|
||||
print(f"Stages Completed: {stats['stages_completed']}/{stats['total_stages']}")
|
||||
|
||||
if 'total_duration_formatted' in stats:
|
||||
print(f"Total Duration: {stats['total_duration_formatted']}")
|
||||
|
||||
# Stage results
|
||||
for stage_name, stage_result in stats.get("stage_results", {}).items():
|
||||
if stage_result.get("success"):
|
||||
print(f"\n✅ {stage_name.title()} Stage:")
|
||||
for key, value in stage_result.items():
|
||||
if key != "success" and not key.startswith("error"):
|
||||
print(f" • {key}: {value}")
|
||||
else:
|
||||
print(f"\n❌ {stage_name.title()} Stage: {stage_result.get('error', 'Unknown error')}")
|
||||
|
||||
# Repository information
|
||||
build_result = stats.get("stage_results", {}).get("build", {})
|
||||
if build_result.get("success"):
|
||||
repo_path = build_result.get("repository_path")
|
||||
print(f"\n🎯 Final Repository: {repo_path}")
|
||||
print("Try these commands:")
|
||||
print(f" cd {repo_path}")
|
||||
print(" git log --oneline")
|
||||
print(" git blame README.md")
|
||||
|
||||
if stats.get("errors"):
|
||||
print(f"\n⚠️ Errors encountered: {len(stats['errors'])}")
|
||||
for error in stats["errors"]:
|
||||
print(f" • {error}")
|
||||
|
||||
sys.exit(0 if success else 1)
|
||||
|
||||
except KeyboardInterrupt:
|
||||
print("\n\n⚠️ Pipeline interrupted by user")
|
||||
sys.exit(1)
|
||||
|
||||
except Exception as e:
|
||||
print(f"\n❌ Pipeline failed with error: {e}")
|
||||
logger.error(f"Unhandled pipeline error: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
855
migrate_to_datastore.py
Normal file
855
migrate_to_datastore.py
Normal file
@@ -0,0 +1,855 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
USC Git Blame Data Migrator
|
||||
|
||||
Processes cached raw data from multiple sources into normalized JSON datastore:
|
||||
|
||||
1. Parses House US Code HTML releases using semantic field extraction
|
||||
2. Normalizes Congress.gov API data with Pydantic validation
|
||||
3. Cross-references bills to public laws to USC sections
|
||||
4. Validates data integrity and builds comprehensive indexes
|
||||
5. Migrates to production-ready normalized datastore
|
||||
|
||||
Architecture: Download → Cache → **Migrate** → Plan → Build
|
||||
This script handles the second step: raw data normalization and validation.
|
||||
"""
|
||||
|
||||
import json
|
||||
import zipfile
|
||||
import re
|
||||
from pathlib import Path
|
||||
from datetime import datetime, date
|
||||
from typing import Dict, List, Optional, Tuple, Any
|
||||
from dataclasses import dataclass
|
||||
import logging
|
||||
from html.parser import HTMLParser
|
||||
import html
|
||||
|
||||
# Import our existing models and datastore
|
||||
from models import Sponsor
|
||||
from datastore import USCodeDataStore
|
||||
from download_cache import CacheManager
|
||||
|
||||
# Configure logging
|
||||
logs_dir = Path('logs')
|
||||
logs_dir.mkdir(exist_ok=True)
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
|
||||
handlers=[
|
||||
logging.FileHandler(logs_dir / 'migrate_to_datastore.log'),
|
||||
logging.StreamHandler()
|
||||
]
|
||||
)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class USCSection:
|
||||
"""Represents an individual USC section extracted from HTML"""
|
||||
title_num: int # 42 (Public Health and Welfare)
|
||||
chapter_num: Optional[str] # "6A" (can have letters)
|
||||
section_num: str # "280g-15" (handles subsection numbering)
|
||||
heading: str # Clean section title
|
||||
statutory_text: str # Normalized legal text
|
||||
source_credit: str # Original enactment attribution
|
||||
amendment_history: Optional[List[str]] = None # Amendment notes
|
||||
cross_references: Optional[List[str]] = None # References to other sections
|
||||
enacted_through: str = "" # Which public law this version reflects
|
||||
|
||||
def __post_init__(self):
|
||||
if self.amendment_history is None:
|
||||
self.amendment_history = []
|
||||
if self.cross_references is None:
|
||||
self.cross_references = []
|
||||
|
||||
@property
|
||||
def section_id(self) -> str:
|
||||
"""Unique identifier for this section"""
|
||||
chapter_part = f"-{self.chapter_num}" if self.chapter_num else ""
|
||||
return f"{self.title_num}{chapter_part}-{self.section_num}"
|
||||
|
||||
@property
|
||||
def file_path(self) -> str:
|
||||
"""File path for hierarchical git structure"""
|
||||
title_name = f"Title-{self.title_num:02d}"
|
||||
if self.chapter_num:
|
||||
chapter_name = f"Chapter-{self.chapter_num}"
|
||||
return f"{title_name}/{chapter_name}/Section-{self.section_num.replace('.', '-')}.md"
|
||||
else:
|
||||
return f"{title_name}/Section-{self.section_num.replace('.', '-')}.md"
|
||||
|
||||
|
||||
@dataclass
|
||||
class ParsedBillData:
|
||||
"""Normalized bill data from Congress.gov API"""
|
||||
congress: int
|
||||
bill_type: str # "hr", "s", etc.
|
||||
bill_number: int
|
||||
title: str
|
||||
sponsor: Optional[Dict[str, Any]]
|
||||
cosponsors: List[Dict[str, Any]]
|
||||
committees: List[Dict[str, Any]]
|
||||
amendments: List[Dict[str, Any]]
|
||||
related_bills: List[Dict[str, Any]]
|
||||
public_law: Optional[str] # "119-001" if this bill became a public law
|
||||
enacted_date: Optional[date]
|
||||
|
||||
|
||||
class USCHTMLParser(HTMLParser):
|
||||
"""Parse USC HTML files using semantic field markers"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.reset_parser_state()
|
||||
|
||||
def reset_parser_state(self):
|
||||
"""Reset parser state for new document"""
|
||||
self.current_section = None
|
||||
self.sections = []
|
||||
self.in_statute_field = False
|
||||
self.in_sourcecredit_field = False
|
||||
self.in_notes_field = False
|
||||
self.current_text = ""
|
||||
self.current_tag = None
|
||||
self.section_data = {}
|
||||
|
||||
def handle_comment(self, data: str):
|
||||
"""Handle HTML comments that contain semantic information"""
|
||||
data = data.strip()
|
||||
|
||||
# Parse itempath comments for section structure
|
||||
if data.startswith("itempath:/"):
|
||||
self._parse_itempath(data)
|
||||
elif data.startswith("expcite:"):
|
||||
self._parse_expcite(data)
|
||||
elif data.startswith("field-start:"):
|
||||
self._handle_field_start(data)
|
||||
elif data.startswith("field-end:"):
|
||||
self._handle_field_end(data)
|
||||
elif data.startswith("AUTHORITIES-LAWS-ENACTED-THROUGH:"):
|
||||
self._parse_enacted_through(data)
|
||||
|
||||
def _parse_itempath(self, data: str):
|
||||
"""Parse itempath to extract section structure"""
|
||||
# Examples:
|
||||
# itempath:/010/CHAPTER 1/Sec. 1
|
||||
# itempath:/042/CHAPTER 6A/SUBCHAPTER II/Part A/Sec. 280g-15
|
||||
|
||||
path_match = re.search(r"itempath:/(\d+)(?:/CHAPTER\s+([^/]+))?(?:/[^/]*)*?(?:/Sec\.\s+(.+))?", data)
|
||||
if path_match:
|
||||
title_num = int(path_match.group(1))
|
||||
chapter_num = path_match.group(2)
|
||||
section_num = path_match.group(3)
|
||||
|
||||
if section_num: # This is a section
|
||||
self.section_data = {
|
||||
"title_num": title_num,
|
||||
"chapter_num": chapter_num,
|
||||
"section_num": section_num.strip(),
|
||||
"heading": "",
|
||||
"statutory_text": "",
|
||||
"source_credit": "",
|
||||
"amendment_history": [],
|
||||
"cross_references": [],
|
||||
"enacted_through": ""
|
||||
}
|
||||
|
||||
def _parse_expcite(self, data: str):
|
||||
"""Parse expcite for additional context"""
|
||||
# Example: expcite:TITLE 42-PUBLIC HEALTH AND WELFARE!@!CHAPTER 6A-PUBLIC HEALTH SERVICE!@!Sec. 280g-15
|
||||
pass # Additional parsing if needed
|
||||
|
||||
def _parse_enacted_through(self, data: str):
|
||||
"""Parse enacted-through info"""
|
||||
# Example: AUTHORITIES-LAWS-ENACTED-THROUGH:119-1 (01/29/2025)
|
||||
match = re.search(r"AUTHORITIES-LAWS-ENACTED-THROUGH:(\d+-\d+)", data)
|
||||
if match and self.section_data:
|
||||
self.section_data["enacted_through"] = match.group(1)
|
||||
|
||||
def _handle_field_start(self, data: str):
|
||||
"""Handle field start markers"""
|
||||
if "statute" in data:
|
||||
self.in_statute_field = True
|
||||
self.current_text = ""
|
||||
elif "sourcecredit" in data:
|
||||
self.in_sourcecredit_field = True
|
||||
self.current_text = ""
|
||||
elif "notes" in data or "amendment-note" in data:
|
||||
self.in_notes_field = True
|
||||
self.current_text = ""
|
||||
|
||||
def _handle_field_end(self, data: str):
|
||||
"""Handle field end markers"""
|
||||
if "statute" in data and self.in_statute_field:
|
||||
if self.section_data:
|
||||
self.section_data["statutory_text"] = self._clean_text(self.current_text)
|
||||
self.in_statute_field = False
|
||||
elif "sourcecredit" in data and self.in_sourcecredit_field:
|
||||
if self.section_data:
|
||||
self.section_data["source_credit"] = self._clean_text(self.current_text)
|
||||
self.in_sourcecredit_field = False
|
||||
elif ("notes" in data or "amendment-note" in data) and self.in_notes_field:
|
||||
if self.section_data and self.current_text.strip():
|
||||
self.section_data["amendment_history"].append(self._clean_text(self.current_text))
|
||||
self.in_notes_field = False
|
||||
|
||||
def handle_starttag(self, tag: str, attrs: List[Tuple[str, str]]):
|
||||
"""Handle HTML start tags"""
|
||||
self.current_tag = tag
|
||||
|
||||
# Extract section headings from section-head class
|
||||
if tag == "h3":
|
||||
for attr_name, attr_value in attrs:
|
||||
if attr_name == "class" and "section-head" in attr_value:
|
||||
self.current_text = ""
|
||||
|
||||
def handle_endtag(self, tag: str):
|
||||
"""Handle HTML end tags"""
|
||||
if tag == "h3" and self.section_data and self.current_text.strip():
|
||||
# Extract section heading
|
||||
heading_text = self._clean_text(self.current_text)
|
||||
# Remove section number prefix (e.g., "§1. " -> "")
|
||||
heading_clean = re.sub(r"^§\s*[\d\w\-\.]+\.\s*", "", heading_text)
|
||||
self.section_data["heading"] = heading_clean
|
||||
|
||||
# Finalize current section if we have complete data
|
||||
if (self.section_data.get("title_num") and
|
||||
self.section_data.get("section_num") and
|
||||
self.section_data.get("statutory_text")):
|
||||
|
||||
section = USCSection(**self.section_data)
|
||||
self.sections.append(section)
|
||||
self.section_data = {}
|
||||
|
||||
self.current_tag = None
|
||||
|
||||
def handle_data(self, data: str):
|
||||
"""Handle text content"""
|
||||
if (self.in_statute_field or self.in_sourcecredit_field or
|
||||
self.in_notes_field or self.current_tag == "h3"):
|
||||
self.current_text += data
|
||||
|
||||
def _clean_text(self, text: str) -> str:
|
||||
"""Clean and normalize text content"""
|
||||
# Decode HTML entities
|
||||
text = html.unescape(text)
|
||||
|
||||
# Normalize whitespace
|
||||
text = re.sub(r'\s+', ' ', text.strip())
|
||||
|
||||
# Convert HTML entities to proper unicode
|
||||
text = text.replace("—", "—")
|
||||
text = text.replace("“", """)
|
||||
text = text.replace("”", """)
|
||||
text = text.replace(" ", " ")
|
||||
|
||||
return text
|
||||
|
||||
def parse_html_file(self, html_content: str) -> List[USCSection]:
|
||||
"""Parse complete HTML file and return extracted sections"""
|
||||
self.reset_parser_state()
|
||||
self.feed(html_content)
|
||||
return self.sections
|
||||
|
||||
|
||||
class DataMigrator:
|
||||
"""
|
||||
Migrates raw cached data into normalized JSON datastore
|
||||
|
||||
Processes:
|
||||
- House USC HTML releases -> USCSection objects
|
||||
- Congress.gov API data -> Normalized bill data
|
||||
- Cross-references and validation
|
||||
- Integration with existing datastore
|
||||
"""
|
||||
|
||||
def __init__(self, cache_dir: Path = Path("download_cache"), force: bool = False):
|
||||
self.cache_manager = CacheManager(cache_dir)
|
||||
self.datastore = USCodeDataStore()
|
||||
self.html_parser = USCHTMLParser()
|
||||
self.force = force # Force re-migration even if output exists
|
||||
|
||||
# Migration statistics
|
||||
self.stats = {
|
||||
"html_files_processed": 0,
|
||||
"usc_sections_extracted": 0,
|
||||
"api_bills_processed": 0,
|
||||
"cross_references_resolved": 0,
|
||||
"validation_errors": 0,
|
||||
"files_skipped": 0,
|
||||
"migration_start_time": datetime.now()
|
||||
}
|
||||
|
||||
def migrate_house_html_data(self, public_laws: List[str]) -> Dict[str, List[USCSection]]:
|
||||
"""
|
||||
Migrate House USC HTML releases to structured section data
|
||||
|
||||
Args:
|
||||
public_laws: List of public law IDs (e.g., ["119-001", "119-004"])
|
||||
|
||||
Returns:
|
||||
Dict mapping public law -> list of USC sections
|
||||
"""
|
||||
logger.info(f"🔄 Migrating House HTML data for {len(public_laws)} public laws")
|
||||
|
||||
all_sections = {}
|
||||
metadata_path = Path("data/usc_sections")
|
||||
|
||||
for law_id in public_laws:
|
||||
# Check if output already exists (idempotency)
|
||||
sections_file = metadata_path / f"{law_id}.json"
|
||||
if sections_file.exists() and not self.force:
|
||||
logger.info(f"✅ Skipping HTML migration for {law_id} - output exists")
|
||||
self.stats["files_skipped"] += 1
|
||||
# Load existing sections for return value
|
||||
try:
|
||||
with open(sections_file, 'r') as f:
|
||||
existing_data = json.load(f)
|
||||
# Convert back to USCSection objects for consistency
|
||||
sections = []
|
||||
for section_data in existing_data.get('sections', []):
|
||||
section = USCSection(
|
||||
section_id=section_data['section_id'],
|
||||
file_path=section_data['file_path'],
|
||||
title_num=section_data['title_num'],
|
||||
chapter_num=section_data['chapter_num'],
|
||||
section_num=section_data['section_num'],
|
||||
heading=section_data['heading'],
|
||||
statutory_text=section_data['statutory_text'],
|
||||
source_credit=section_data['source_credit'],
|
||||
amendment_history=section_data['amendment_history'],
|
||||
cross_references=section_data['cross_references'],
|
||||
enacted_through=section_data['enacted_through']
|
||||
)
|
||||
sections.append(section)
|
||||
all_sections[law_id] = sections
|
||||
except Exception as e:
|
||||
logger.warning(f"⚠️ Error loading existing sections for {law_id}: {e}")
|
||||
continue
|
||||
congress, law_num = law_id.split("-")
|
||||
cache_key = f"house_usc_{congress}_{law_num}"
|
||||
|
||||
if not self.cache_manager.is_cached(cache_key):
|
||||
logger.warning(f"⚠️ No cached HTML data for {law_id}")
|
||||
continue
|
||||
|
||||
zip_path = self.cache_manager.get_cached_path(cache_key)
|
||||
sections = self._extract_sections_from_zip(zip_path, law_id)
|
||||
|
||||
if sections:
|
||||
all_sections[law_id] = sections
|
||||
self.stats["usc_sections_extracted"] += len(sections)
|
||||
logger.info(f"✅ Extracted {len(sections)} sections from {law_id}")
|
||||
else:
|
||||
logger.warning(f"⚠️ No sections extracted from {law_id}")
|
||||
|
||||
self.stats["html_files_processed"] = len(all_sections)
|
||||
logger.info(f"📊 HTML migration complete: {self.stats['usc_sections_extracted']} total sections")
|
||||
|
||||
return all_sections
|
||||
|
||||
def _extract_sections_from_zip(self, zip_path: Path, law_id: str) -> List[USCSection]:
|
||||
"""Extract USC sections from downloaded ZIP file"""
|
||||
sections = []
|
||||
|
||||
try:
|
||||
with zipfile.ZipFile(zip_path, 'r') as zip_file:
|
||||
html_files = [name for name in zip_file.namelist() if name.endswith('.htm')]
|
||||
|
||||
for html_file in html_files:
|
||||
html_content = zip_file.read(html_file).decode('utf-8', errors='ignore')
|
||||
file_sections = self.html_parser.parse_html_file(html_content)
|
||||
|
||||
# Set enacted_through for all sections from this release
|
||||
for section in file_sections:
|
||||
if not section.enacted_through:
|
||||
section.enacted_through = law_id
|
||||
|
||||
sections.extend(file_sections)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"❌ Error extracting sections from {zip_path}: {e}")
|
||||
|
||||
return sections
|
||||
|
||||
def migrate_congress_api_data(self, public_laws: List[str]) -> Dict[str, ParsedBillData]:
|
||||
"""
|
||||
Migrate Congress.gov API data to normalized bill structures
|
||||
|
||||
Args:
|
||||
public_laws: List of public law IDs
|
||||
|
||||
Returns:
|
||||
Dict mapping public law -> normalized bill data
|
||||
"""
|
||||
logger.info(f"🔄 Migrating Congress.gov API data for {len(public_laws)} public laws")
|
||||
|
||||
normalized_bills = {}
|
||||
|
||||
for law_id in public_laws:
|
||||
congress, law_num = law_id.split("-")
|
||||
|
||||
# Look for cached bill details
|
||||
search_cache_key = f"bill_search_{congress}_{law_num.zfill(3)}"
|
||||
|
||||
if not self.cache_manager.is_cached(search_cache_key):
|
||||
logger.warning(f"⚠️ No cached API data for {law_id}")
|
||||
continue
|
||||
|
||||
# Load bill search results
|
||||
search_path = self.cache_manager.get_cached_path(search_cache_key)
|
||||
try:
|
||||
with open(search_path, 'r') as f:
|
||||
bill_info = json.load(f)
|
||||
|
||||
# Load full bill details if available
|
||||
bill_type = bill_info.get('bill_type', '').lower()
|
||||
bill_number = bill_info.get('bill_number')
|
||||
|
||||
if bill_type and bill_number:
|
||||
details_cache_key = f"bill_details_{congress}_{bill_type}_{bill_number}"
|
||||
|
||||
if self.cache_manager.is_cached(details_cache_key):
|
||||
details_path = self.cache_manager.get_cached_path(details_cache_key)
|
||||
with open(details_path, 'r') as f:
|
||||
bill_details = json.load(f)
|
||||
|
||||
# Normalize the bill data
|
||||
normalized_bill = self._normalize_bill_data(bill_details, law_id)
|
||||
normalized_bills[law_id] = normalized_bill
|
||||
|
||||
self.stats["api_bills_processed"] += 1
|
||||
logger.info(f"✅ Normalized API data for {law_id}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"❌ Error processing API data for {law_id}: {e}")
|
||||
self.stats["validation_errors"] += 1
|
||||
|
||||
logger.info(f"📊 API migration complete: {len(normalized_bills)} bills normalized")
|
||||
return normalized_bills
|
||||
|
||||
def _normalize_bill_data(self, bill_details: Dict[str, Any], law_id: str) -> ParsedBillData:
|
||||
"""Normalize raw bill data from Congress.gov API"""
|
||||
|
||||
basic_info = bill_details.get('details', {})
|
||||
|
||||
# Extract basic bill information
|
||||
congress = basic_info.get('congress', 0)
|
||||
bill_type = basic_info.get('type', '').lower()
|
||||
bill_number = basic_info.get('number', 0)
|
||||
title = basic_info.get('title', '')
|
||||
|
||||
# Extract sponsor information
|
||||
sponsor = None
|
||||
sponsor_data = basic_info.get('sponsors', [])
|
||||
if sponsor_data and len(sponsor_data) > 0:
|
||||
sponsor = sponsor_data[0] # Primary sponsor
|
||||
|
||||
# Extract cosponsors
|
||||
cosponsors = bill_details.get('cosponsors', [])
|
||||
|
||||
# Extract committee information
|
||||
committees = bill_details.get('committees', [])
|
||||
|
||||
# Extract amendments
|
||||
amendments = bill_details.get('amendments', [])
|
||||
|
||||
# Extract related bills
|
||||
related_bills = bill_details.get('related_bills', [])
|
||||
|
||||
# Extract enactment information
|
||||
enacted_date = None
|
||||
public_law = law_id
|
||||
|
||||
# Try to parse enacted date from basic info
|
||||
if 'becamelaw' in basic_info:
|
||||
became_law = basic_info['becamelaw']
|
||||
if isinstance(became_law, str):
|
||||
try:
|
||||
enacted_date = datetime.strptime(became_law, '%Y-%m-%d').date()
|
||||
except ValueError:
|
||||
# Try other date formats
|
||||
for date_format in ['%Y-%m-%d', '%m/%d/%Y', '%B %d, %Y']:
|
||||
try:
|
||||
enacted_date = datetime.strptime(became_law, date_format).date()
|
||||
break
|
||||
except ValueError:
|
||||
continue
|
||||
|
||||
return ParsedBillData(
|
||||
congress=congress,
|
||||
bill_type=bill_type,
|
||||
bill_number=bill_number,
|
||||
title=title,
|
||||
sponsor=sponsor,
|
||||
cosponsors=cosponsors,
|
||||
committees=committees,
|
||||
amendments=amendments,
|
||||
related_bills=related_bills,
|
||||
public_law=public_law,
|
||||
enacted_date=enacted_date
|
||||
)
|
||||
|
||||
def cross_reference_and_validate(self,
|
||||
usc_sections: Dict[str, List[USCSection]],
|
||||
bill_data: Dict[str, ParsedBillData]) -> Dict[str, Any]:
|
||||
"""
|
||||
Cross-reference USC sections with bill data and validate relationships
|
||||
|
||||
Args:
|
||||
usc_sections: Dict of public law -> USC sections
|
||||
bill_data: Dict of public law -> normalized bill data
|
||||
|
||||
Returns:
|
||||
Dict with validation results and cross-reference mappings
|
||||
"""
|
||||
logger.info("🔄 Cross-referencing and validating data relationships")
|
||||
|
||||
validation_results = {
|
||||
"total_laws_processed": len(set(list(usc_sections.keys()) + list(bill_data.keys()))),
|
||||
"laws_with_both_html_and_api": 0,
|
||||
"laws_missing_html": [],
|
||||
"laws_missing_api": [],
|
||||
"section_title_distribution": {},
|
||||
"sponsor_attribution_success": 0,
|
||||
"validation_errors": []
|
||||
}
|
||||
|
||||
all_laws = set(list(usc_sections.keys()) + list(bill_data.keys()))
|
||||
|
||||
for law_id in all_laws:
|
||||
has_html = law_id in usc_sections
|
||||
has_api = law_id in bill_data
|
||||
|
||||
if has_html and has_api:
|
||||
validation_results["laws_with_both_html_and_api"] += 1
|
||||
|
||||
# Cross-reference sponsor data
|
||||
sections = usc_sections[law_id]
|
||||
bill = bill_data[law_id]
|
||||
|
||||
if bill.sponsor:
|
||||
validation_results["sponsor_attribution_success"] += 1
|
||||
|
||||
# Track section title distribution
|
||||
for section in sections:
|
||||
title_key = f"Title-{section.title_num}"
|
||||
validation_results["section_title_distribution"][title_key] = \
|
||||
validation_results["section_title_distribution"].get(title_key, 0) + 1
|
||||
|
||||
elif not has_html:
|
||||
validation_results["laws_missing_html"].append(law_id)
|
||||
elif not has_api:
|
||||
validation_results["laws_missing_api"].append(law_id)
|
||||
|
||||
# Validate USC section data quality
|
||||
total_sections = sum(len(sections) for sections in usc_sections.values())
|
||||
sections_with_text = sum(1 for sections in usc_sections.values()
|
||||
for section in sections if section.statutory_text.strip())
|
||||
|
||||
validation_results.update({
|
||||
"total_sections_extracted": total_sections,
|
||||
"sections_with_statutory_text": sections_with_text,
|
||||
"text_extraction_rate": sections_with_text / total_sections if total_sections > 0 else 0
|
||||
})
|
||||
|
||||
self.stats["cross_references_resolved"] = validation_results["laws_with_both_html_and_api"]
|
||||
|
||||
logger.info("📊 Cross-reference complete:")
|
||||
logger.info(f" • {validation_results['laws_with_both_html_and_api']} laws with complete data")
|
||||
logger.info(f" • {len(validation_results['laws_missing_html'])} laws missing HTML")
|
||||
logger.info(f" • {len(validation_results['laws_missing_api'])} laws missing API data")
|
||||
logger.info(f" • {validation_results['text_extraction_rate']:.2%} text extraction success rate")
|
||||
|
||||
return validation_results
|
||||
|
||||
def integrate_with_datastore(self,
|
||||
usc_sections: Dict[str, List[USCSection]],
|
||||
bill_data: Dict[str, ParsedBillData],
|
||||
validation_results: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""
|
||||
Integrate migrated data with existing datastore
|
||||
|
||||
Args:
|
||||
usc_sections: Extracted USC sections
|
||||
bill_data: Normalized bill data
|
||||
validation_results: Cross-reference validation results
|
||||
|
||||
Returns:
|
||||
Integration statistics
|
||||
"""
|
||||
logger.info("🔄 Integrating migrated data with existing datastore")
|
||||
|
||||
integration_stats = {
|
||||
"existing_laws_in_datastore": self.datastore.public_laws.count(),
|
||||
"new_sections_added": 0,
|
||||
"enhanced_laws_with_api_data": 0,
|
||||
"sponsor_profiles_created": 0,
|
||||
"integration_errors": []
|
||||
}
|
||||
|
||||
# Create sponsor profiles from bill data
|
||||
unique_sponsors = set()
|
||||
for bill in bill_data.values():
|
||||
if bill.sponsor and 'bioguideId' in bill.sponsor:
|
||||
bioguide_id = bill.sponsor['bioguideId']
|
||||
if bioguide_id not in unique_sponsors:
|
||||
try:
|
||||
sponsor = self._create_sponsor_from_api_data(bill.sponsor)
|
||||
if sponsor:
|
||||
self.datastore.sponsors.save(bioguide_id, sponsor)
|
||||
unique_sponsors.add(bioguide_id)
|
||||
integration_stats["sponsor_profiles_created"] += 1
|
||||
except Exception as e:
|
||||
integration_stats["integration_errors"].append(f"Sponsor creation error: {e}")
|
||||
|
||||
# Save USC sections as metadata for future git processing
|
||||
sections_metadata = {}
|
||||
for law_id, sections in usc_sections.items():
|
||||
sections_data = []
|
||||
for section in sections:
|
||||
sections_data.append({
|
||||
"section_id": section.section_id,
|
||||
"file_path": section.file_path,
|
||||
"title_num": section.title_num,
|
||||
"chapter_num": section.chapter_num,
|
||||
"section_num": section.section_num,
|
||||
"heading": section.heading,
|
||||
"statutory_text": section.statutory_text,
|
||||
"source_credit": section.source_credit,
|
||||
"amendment_history": section.amendment_history,
|
||||
"cross_references": section.cross_references,
|
||||
"enacted_through": section.enacted_through
|
||||
})
|
||||
|
||||
sections_metadata[law_id] = {
|
||||
"public_law": law_id,
|
||||
"sections": sections_data,
|
||||
"extracted_at": datetime.now().isoformat(),
|
||||
"section_count": len(sections_data)
|
||||
}
|
||||
|
||||
integration_stats["new_sections_added"] += len(sections_data)
|
||||
|
||||
# Save sections metadata to datastore
|
||||
try:
|
||||
metadata_path = Path("data/usc_sections")
|
||||
metadata_path.mkdir(exist_ok=True)
|
||||
|
||||
for law_id, metadata in sections_metadata.items():
|
||||
sections_file = metadata_path / f"{law_id}.json"
|
||||
|
||||
# Skip if file already exists and not forcing re-migration
|
||||
if sections_file.exists() and not self.force:
|
||||
logger.info(f"✅ Skipping {law_id} - sections file already exists")
|
||||
self.stats["files_skipped"] += 1
|
||||
continue
|
||||
|
||||
with open(sections_file, 'w') as f:
|
||||
json.dump(metadata, f, indent=2, default=str)
|
||||
|
||||
if self.force and sections_file.exists():
|
||||
logger.info(f"🔄 Force-updated sections for {law_id}")
|
||||
else:
|
||||
logger.info(f"💾 Created sections file for {law_id}")
|
||||
|
||||
except Exception as e:
|
||||
integration_stats["integration_errors"].append(f"Sections metadata save error: {e}")
|
||||
|
||||
# Update existing public law records with enhanced API data
|
||||
for law_id, bill in bill_data.items():
|
||||
congress, law_num = law_id.split("-")
|
||||
|
||||
try:
|
||||
# Try to get existing public law record
|
||||
existing_law = self.datastore.get_public_law(int(congress), int(law_num))
|
||||
|
||||
if existing_law and bill.enacted_date:
|
||||
# Update with more accurate enacted date if available
|
||||
if existing_law.enacted_date != bill.enacted_date:
|
||||
existing_law.enacted_date = bill.enacted_date
|
||||
self.datastore.public_laws.save(f"{congress}-{law_num.zfill(3)}", existing_law)
|
||||
integration_stats["enhanced_laws_with_api_data"] += 1
|
||||
|
||||
except Exception as e:
|
||||
integration_stats["integration_errors"].append(f"Law update error for {law_id}: {e}")
|
||||
|
||||
logger.info("📊 Integration complete:")
|
||||
logger.info(f" • {integration_stats['new_sections_added']} USC sections saved")
|
||||
logger.info(f" • {integration_stats['sponsor_profiles_created']} sponsor profiles created")
|
||||
logger.info(f" • {integration_stats['enhanced_laws_with_api_data']} laws enhanced with API data")
|
||||
|
||||
return integration_stats
|
||||
|
||||
def _create_sponsor_from_api_data(self, sponsor_data: Dict[str, Any]) -> Optional[Sponsor]:
|
||||
"""Create Sponsor object from Congress.gov API data"""
|
||||
|
||||
try:
|
||||
bioguide_id = sponsor_data.get('bioguideId', '')
|
||||
if not bioguide_id:
|
||||
return None
|
||||
|
||||
# Extract basic information
|
||||
first_name = sponsor_data.get('firstName', '')
|
||||
last_name = sponsor_data.get('lastName', '')
|
||||
party = sponsor_data.get('party', '')
|
||||
state = sponsor_data.get('state', '')
|
||||
|
||||
# Determine chamber and title
|
||||
chamber = "house" # Default
|
||||
title = "Representative"
|
||||
|
||||
if 'chamber' in sponsor_data:
|
||||
chamber_name = sponsor_data['chamber'].lower()
|
||||
if 'senate' in chamber_name:
|
||||
chamber = "senate"
|
||||
title = "Senator"
|
||||
|
||||
# Map party to enum value
|
||||
from models import PoliticalParty, CongressionalChamber
|
||||
|
||||
party_enum = PoliticalParty.UNKNOWN
|
||||
if party == "D":
|
||||
party_enum = PoliticalParty.DEMOCRATIC
|
||||
elif party == "R":
|
||||
party_enum = PoliticalParty.REPUBLICAN
|
||||
elif party == "I":
|
||||
party_enum = PoliticalParty.INDEPENDENT
|
||||
|
||||
chamber_enum = CongressionalChamber.HOUSE
|
||||
if chamber == "senate":
|
||||
chamber_enum = CongressionalChamber.SENATE
|
||||
|
||||
# Parse district number
|
||||
district_num = None
|
||||
district_str = sponsor_data.get('district', '')
|
||||
if district_str and district_str.isdigit():
|
||||
district_num = int(district_str)
|
||||
|
||||
# Create sponsor object
|
||||
sponsor = Sponsor(
|
||||
bioguide_id=bioguide_id,
|
||||
title=title,
|
||||
first_name=first_name,
|
||||
last_name=last_name,
|
||||
full_name=f"{first_name} {last_name}".strip(),
|
||||
party=party_enum,
|
||||
state=state,
|
||||
district=district_num,
|
||||
chamber=chamber_enum
|
||||
)
|
||||
|
||||
return sponsor
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"❌ Error creating sponsor from API data: {e}")
|
||||
return None
|
||||
|
||||
def get_migration_statistics(self) -> Dict[str, Any]:
|
||||
"""Get comprehensive migration statistics"""
|
||||
|
||||
end_time = datetime.now()
|
||||
duration = end_time - self.stats["migration_start_time"]
|
||||
|
||||
return {
|
||||
"migration_duration_seconds": duration.total_seconds(),
|
||||
"migration_duration_formatted": str(duration),
|
||||
**self.stats,
|
||||
"migration_completed_at": end_time.isoformat()
|
||||
}
|
||||
|
||||
def run_full_migration(self, public_laws: List[str]) -> Dict[str, Any]:
|
||||
"""
|
||||
Run complete migration pipeline
|
||||
|
||||
Args:
|
||||
public_laws: List of public law IDs to migrate
|
||||
|
||||
Returns:
|
||||
Complete migration results with statistics
|
||||
"""
|
||||
logger.info(f"🚀 Starting full migration for {len(public_laws)} public laws")
|
||||
|
||||
results = {
|
||||
"public_laws_requested": public_laws,
|
||||
"migration_phases": {}
|
||||
}
|
||||
|
||||
# Phase 1: Migrate House HTML data
|
||||
logger.info("📋 Phase 1: House HTML Data Migration")
|
||||
usc_sections = self.migrate_house_html_data(public_laws)
|
||||
results["migration_phases"]["html_migration"] = {
|
||||
"laws_processed": len(usc_sections),
|
||||
"sections_extracted": sum(len(sections) for sections in usc_sections.values())
|
||||
}
|
||||
|
||||
# Phase 2: Migrate Congress.gov API data
|
||||
logger.info("📋 Phase 2: Congress.gov API Data Migration")
|
||||
bill_data = self.migrate_congress_api_data(public_laws)
|
||||
results["migration_phases"]["api_migration"] = {
|
||||
"bills_processed": len(bill_data)
|
||||
}
|
||||
|
||||
# Phase 3: Cross-reference and validate
|
||||
logger.info("📋 Phase 3: Cross-Reference and Validation")
|
||||
validation_results = self.cross_reference_and_validate(usc_sections, bill_data)
|
||||
results["migration_phases"]["validation"] = validation_results
|
||||
|
||||
# Phase 4: Integrate with datastore
|
||||
logger.info("📋 Phase 4: Datastore Integration")
|
||||
integration_results = self.integrate_with_datastore(usc_sections, bill_data, validation_results)
|
||||
results["migration_phases"]["integration"] = integration_results
|
||||
|
||||
# Final statistics
|
||||
migration_stats = self.get_migration_statistics()
|
||||
results["migration_statistics"] = migration_stats
|
||||
|
||||
logger.info("🎉 Full migration complete!")
|
||||
logger.info("📊 Summary:")
|
||||
logger.info(f" • Duration: {migration_stats['migration_duration_formatted']}")
|
||||
logger.info(f" • HTML files: {migration_stats['html_files_processed']}")
|
||||
logger.info(f" • USC sections: {migration_stats['usc_sections_extracted']}")
|
||||
logger.info(f" • API bills: {migration_stats['api_bills_processed']}")
|
||||
logger.info(f" • Cross-references: {migration_stats['cross_references_resolved']}")
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def main():
|
||||
"""Example usage of the data migrator"""
|
||||
|
||||
# Initialize migrator
|
||||
migrator = DataMigrator()
|
||||
|
||||
# Example: Migrate recent public laws
|
||||
public_laws = ["119-001", "119-004", "119-012", "119-018", "119-023", "119-026"]
|
||||
|
||||
logger.info("🚀 Starting USC data migration process")
|
||||
|
||||
# Run full migration
|
||||
results = migrator.run_full_migration(public_laws)
|
||||
|
||||
# Display results
|
||||
print("\n" + "="*60)
|
||||
print("📊 MIGRATION RESULTS")
|
||||
print("="*60)
|
||||
|
||||
for phase_name, phase_results in results["migration_phases"].items():
|
||||
print(f"\n{phase_name.upper()}:")
|
||||
for key, value in phase_results.items():
|
||||
if isinstance(value, list) and len(value) > 10:
|
||||
print(f" {key}: {len(value)} items")
|
||||
elif isinstance(value, float):
|
||||
print(f" {key}: {value:.2%}" if "rate" in key else f" {key}: {value:.2f}")
|
||||
else:
|
||||
print(f" {key}: {value}")
|
||||
|
||||
stats = results["migration_statistics"]
|
||||
print(f"\n⏱️ Total Duration: {stats['migration_duration_formatted']}")
|
||||
print("✅ Migration completed successfully!")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
300
models.py
Normal file
300
models.py
Normal file
@@ -0,0 +1,300 @@
|
||||
"""
|
||||
Data models for US Code git repository system.
|
||||
Provides normalized, validated structures for all legislative data.
|
||||
"""
|
||||
|
||||
from datetime import datetime, date
|
||||
from typing import List, Optional, Dict, Any, Literal
|
||||
from pathlib import Path
|
||||
from pydantic import BaseModel, Field, field_validator, HttpUrl
|
||||
from enum import Enum
|
||||
|
||||
|
||||
class CongressionalChamber(str, Enum):
|
||||
"""Congressional chambers"""
|
||||
HOUSE = "House"
|
||||
SENATE = "Senate"
|
||||
|
||||
|
||||
class PoliticalParty(str, Enum):
|
||||
"""Political parties (expandable)"""
|
||||
DEMOCRATIC = "D"
|
||||
REPUBLICAN = "R"
|
||||
INDEPENDENT = "I"
|
||||
LIBERTARIAN = "L"
|
||||
GREEN = "G"
|
||||
UNKNOWN = "Unknown"
|
||||
|
||||
|
||||
class BillType(str, Enum):
|
||||
"""Types of congressional bills"""
|
||||
HOUSE_BILL = "HR"
|
||||
SENATE_BILL = "S"
|
||||
HOUSE_JOINT_RESOLUTION = "HJR"
|
||||
SENATE_JOINT_RESOLUTION = "SJR"
|
||||
HOUSE_CONCURRENT_RESOLUTION = "HCR"
|
||||
SENATE_CONCURRENT_RESOLUTION = "SCR"
|
||||
HOUSE_RESOLUTION = "HRES"
|
||||
SENATE_RESOLUTION = "SRES"
|
||||
|
||||
|
||||
class LegislativeAction(BaseModel):
|
||||
"""A single legislative action on a bill"""
|
||||
date: date
|
||||
action_code: Optional[str] = None
|
||||
text: str
|
||||
chamber: Optional[CongressionalChamber] = None
|
||||
|
||||
class Config:
|
||||
use_enum_values = True
|
||||
|
||||
|
||||
class Sponsor(BaseModel):
|
||||
"""Congressional member who sponsors legislation"""
|
||||
bioguide_id: Optional[str] = Field(None, description="Biographical Directory ID")
|
||||
title: str = Field(..., description="Rep. or Sen.")
|
||||
first_name: str
|
||||
last_name: str
|
||||
full_name: str
|
||||
party: PoliticalParty
|
||||
state: str = Field(..., min_length=2, max_length=2, description="Two-letter state code")
|
||||
district: Optional[int] = Field(None, description="House district number (if applicable)")
|
||||
chamber: CongressionalChamber
|
||||
|
||||
# Computed fields
|
||||
@property
|
||||
def email(self) -> str:
|
||||
"""Generate congressional email address"""
|
||||
first = self.first_name.lower().replace(" ", "")
|
||||
last = self.last_name.lower().replace(" ", "")
|
||||
domain = "house.gov" if self.chamber == CongressionalChamber.HOUSE else "senate.gov"
|
||||
return f"{first}.{last}@{domain}"
|
||||
|
||||
@property
|
||||
def formal_name(self) -> str:
|
||||
"""Full formal name with title and party"""
|
||||
return f"{self.title} {self.full_name} ({self.party}-{self.state})"
|
||||
|
||||
class Config:
|
||||
use_enum_values = True
|
||||
|
||||
|
||||
class Bill(BaseModel):
|
||||
"""Congressional bill that may become a public law"""
|
||||
congress: int = Field(..., ge=1, description="Congressional session number")
|
||||
bill_type: BillType
|
||||
number: int = Field(..., ge=1, description="Bill number within type/congress")
|
||||
title: Optional[str] = None
|
||||
short_title: Optional[str] = None
|
||||
|
||||
# Sponsorship
|
||||
primary_sponsor: Optional["Sponsor"] = None
|
||||
cosponsors: List["Sponsor"] = Field(default_factory=lambda: [])
|
||||
|
||||
# Legislative process
|
||||
introduced_date: Optional[date] = None
|
||||
actions: List["LegislativeAction"] = Field(default_factory=lambda: [])
|
||||
|
||||
# External IDs
|
||||
congress_gov_url: Optional[HttpUrl] = None
|
||||
|
||||
@property
|
||||
def bill_id(self) -> str:
|
||||
"""Unique bill identifier"""
|
||||
return f"{self.bill_type.value}{self.number}"
|
||||
|
||||
@property
|
||||
def full_id(self) -> str:
|
||||
"""Full bill identifier with congress"""
|
||||
return f"{self.congress}-{self.bill_id}"
|
||||
|
||||
class Config:
|
||||
use_enum_values = True
|
||||
|
||||
|
||||
class PublicLaw(BaseModel):
|
||||
"""A bill that has been enacted into public law"""
|
||||
congress: int = Field(..., ge=1, description="Congressional session number")
|
||||
law_number: int = Field(..., ge=1, description="Public law number within congress")
|
||||
|
||||
# Enactment details
|
||||
enacted_date: date
|
||||
signed_date: Optional[date] = None
|
||||
|
||||
# Source bill (if known)
|
||||
source_bill: Optional[Bill] = None
|
||||
|
||||
# US Code impact
|
||||
affected_titles: List[int] = Field(default_factory=lambda: [], description="US Code titles affected")
|
||||
|
||||
# House release point data
|
||||
release_path: str = Field(..., description="House download path (e.g., '119/26not21')")
|
||||
|
||||
# Metadata
|
||||
title: Optional[str] = None
|
||||
description: Optional[str] = None
|
||||
|
||||
@property
|
||||
def public_law_id(self) -> str:
|
||||
"""Standard public law identifier"""
|
||||
return f"PL {self.congress}-{self.law_number}"
|
||||
|
||||
@property
|
||||
def formal_citation(self) -> str:
|
||||
"""Formal legal citation"""
|
||||
return f"Public Law {self.congress}-{self.law_number:03d}"
|
||||
|
||||
@field_validator('affected_titles')
|
||||
@classmethod
|
||||
def validate_titles(cls, v: List[int]) -> List[int]:
|
||||
"""Validate US Code title numbers"""
|
||||
for title in v:
|
||||
if not (1 <= title <= 54):
|
||||
raise ValueError(f"Invalid US Code title: {title}")
|
||||
return sorted(v)
|
||||
|
||||
class Config:
|
||||
use_enum_values = True
|
||||
|
||||
|
||||
class USCodeRelease(BaseModel):
|
||||
"""A specific release of the US Code from House sources"""
|
||||
public_law: PublicLaw
|
||||
|
||||
# Download metadata
|
||||
download_url: str
|
||||
download_date: Optional[datetime] = None
|
||||
file_size: Optional[int] = None
|
||||
file_hash: Optional[str] = Field(None, description="SHA-256 hash")
|
||||
|
||||
# Extraction metadata
|
||||
extracted_date: Optional[datetime] = None
|
||||
extraction_path: Optional[Path] = None
|
||||
file_count: Optional[int] = None
|
||||
|
||||
# Git metadata
|
||||
git_commit_hash: Optional[str] = None
|
||||
git_tag: Optional[str] = None
|
||||
|
||||
@property
|
||||
def release_filename(self) -> str:
|
||||
"""Standard filename for this release"""
|
||||
return self.public_law.release_path.replace("/", "-")
|
||||
|
||||
|
||||
class CongressionalSession(BaseModel):
|
||||
"""Information about a congressional session"""
|
||||
congress: int = Field(..., ge=1)
|
||||
session: Literal[1, 2] = Field(..., description="1st or 2nd session")
|
||||
start_date: date
|
||||
end_date: Optional[date] = None
|
||||
|
||||
# Leadership
|
||||
house_speaker: Optional[Sponsor] = None
|
||||
senate_majority_leader: Optional[Sponsor] = None
|
||||
house_majority_leader: Optional[Sponsor] = None
|
||||
senate_minority_leader: Optional[Sponsor] = None
|
||||
|
||||
# Party control
|
||||
house_majority_party: Optional[PoliticalParty] = None
|
||||
senate_majority_party: Optional[PoliticalParty] = None
|
||||
|
||||
@property
|
||||
def session_id(self) -> str:
|
||||
"""Session identifier"""
|
||||
return f"{self.congress}-{self.session}"
|
||||
|
||||
@property
|
||||
def formal_name(self) -> str:
|
||||
"""Formal session name"""
|
||||
ordinal = "1st" if self.session == 1 else "2nd"
|
||||
return f"{self.congress}th Congress, {ordinal} Session"
|
||||
|
||||
class Config:
|
||||
use_enum_values = True
|
||||
|
||||
|
||||
class GitCommitMetadata(BaseModel):
|
||||
"""Metadata for git commits in the US Code repository"""
|
||||
public_law: PublicLaw
|
||||
|
||||
# Git data
|
||||
commit_hash: str
|
||||
tag_name: str
|
||||
author_name: str
|
||||
author_email: str
|
||||
commit_date: datetime
|
||||
message: str
|
||||
|
||||
# File changes
|
||||
files_changed: int
|
||||
lines_added: int
|
||||
lines_deleted: int
|
||||
|
||||
# Repository state
|
||||
repository_path: Path
|
||||
is_initial_commit: bool = False
|
||||
|
||||
|
||||
class APICache(BaseModel):
|
||||
"""Cache entry for Congress.gov API responses"""
|
||||
cache_key: str
|
||||
congress: int
|
||||
law_number: int
|
||||
|
||||
# Cache metadata
|
||||
cached_date: datetime
|
||||
api_response: Dict[str, Any]
|
||||
sponsor_found: bool = False
|
||||
|
||||
# Extracted sponsor (if found)
|
||||
sponsor: Optional[Sponsor] = None
|
||||
|
||||
|
||||
class RepositoryMetadata(BaseModel):
|
||||
"""Overall metadata for the US Code git repository"""
|
||||
created_date: datetime
|
||||
last_updated: datetime
|
||||
|
||||
# Coverage
|
||||
earliest_law: PublicLaw
|
||||
latest_law: PublicLaw
|
||||
total_laws: int
|
||||
total_commits: int
|
||||
|
||||
# Data sources
|
||||
congress_api_key_used: bool
|
||||
house_source_verified: bool
|
||||
|
||||
# Repository info
|
||||
repository_path: Path
|
||||
total_size: Optional[int] = None
|
||||
|
||||
|
||||
# Utility functions for model creation
|
||||
|
||||
def create_sponsor_from_congress_api(api_data: Dict[str, Any]) -> Sponsor:
|
||||
"""Create Sponsor from Congress.gov API response"""
|
||||
return Sponsor(
|
||||
bioguide_id=api_data.get('bioguideId'),
|
||||
title=api_data.get('title', ''),
|
||||
first_name=api_data.get('firstName', ''),
|
||||
last_name=api_data.get('lastName', ''),
|
||||
full_name=api_data.get('fullName', ''),
|
||||
party=PoliticalParty(api_data.get('party', 'Unknown')),
|
||||
state=api_data.get('state', ''),
|
||||
district=api_data.get('district'),
|
||||
chamber=CongressionalChamber.HOUSE if api_data.get('title') == 'Rep.' else CongressionalChamber.SENATE
|
||||
)
|
||||
|
||||
|
||||
def create_public_law_from_house_data(house_data: Dict[str, Any]) -> PublicLaw:
|
||||
"""Create PublicLaw from House release point data"""
|
||||
return PublicLaw(
|
||||
congress=house_data['congress'],
|
||||
law_number=house_data['law'],
|
||||
enacted_date=datetime.strptime(house_data['date'], '%m/%d/%Y').date(),
|
||||
release_path=house_data['releasePath'],
|
||||
affected_titles=house_data['affectedTitles']
|
||||
)
|
||||
|
||||
17
pyproject.toml
Normal file
17
pyproject.toml
Normal file
@@ -0,0 +1,17 @@
|
||||
[project]
|
||||
name = "gitlaw"
|
||||
version = "0.1.0"
|
||||
description = "Add your description here"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.11"
|
||||
dependencies = [
|
||||
"requests",
|
||||
"python-dateutil",
|
||||
"python-dotenv",
|
||||
"pydantic",
|
||||
]
|
||||
|
||||
[dependency-groups]
|
||||
dev = [
|
||||
"ruff>=0.12.8",
|
||||
]
|
||||
Reference in New Issue
Block a user