1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468"""Comprehensive test suite for timezone consistency across the codebase.
Tests all datetime operations to ensure:
1. All datetimes are timezone-aware (UTC)
2. Datetime serialization preserves timezone information
3. Datetime parsing creates timezone-aware objects
4. Datetime comparisons work correctly across services
5. Edge cases are handled (DST boundaries, UTC midnight, etc.)
"""
from datetime import UTC, datetime, timedelta
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from src.services.approval_service import ApprovalService
from src.services.job_service import JobService
from src.services.queue_service import QueueService
def _extract_lua_mapping(lua_script_mock) -> dict:
"""Extract the field mapping from a Lua script mock call.
Lua script args format: [str(ttl), str(now_ts), job_id, k1, v1, k2, v2, ...]
or for scripts without TTL: [str(now_ts), job_id, k1, v1, k2, v2, ...]
Returns the key-value pairs as a dict.
"""
call_args = lua_script_mock.call_args
args = call_args.kwargs.get("args", call_args[1] if len(call_args) > 1 else [])
# Find where field/value pairs start (after ttl, now_ts, job_id)
# The first 3 args are always ttl, now_ts, job_id for hset_expire_zadd
field_args = args[3:]
mapping = {}
for i in range(0, len(field_args), 2):
mapping[field_args[i]] = field_args[i + 1]
return mapping
class TestDatetimeCreation:
"""Test that all datetime creation uses timezone-aware UTC."""
@pytest.mark.asyncio
async def test_job_service_creates_timezone_aware_datetimes(self, mock_redis):
"""Test JobService.create_job stores timezone-aware timestamps."""
job_service = JobService(mock_redis)
await job_service.create_job(
job_id="test-job-123",
s3_key="temp/test.pdf",
status="pii_scanning"
)
# Extract data from Lua script call
lua_mock = mock_redis.register_script.return_value
stored_data = _extract_lua_mapping(lua_mock)
# Verify created_at is timezone-aware ISO format
created_at_str = stored_data['created_at']
created_at = datetime.fromisoformat(created_at_str.replace("Z", "+00:00"))
# Should be timezone-aware
assert created_at.tzinfo is not None
assert created_at.tzinfo == UTC or created_at.tzinfo.utcoffset(None) == timedelta(0)
# Should be ISO format with timezone info
assert "+" in created_at_str or created_at_str.endswith("Z")
@pytest.mark.asyncio
async def test_job_service_update_uses_timezone_aware_datetimes(self, mock_redis):
"""Test JobService.update_job_status stores timezone-aware timestamps."""
job_service = JobService(mock_redis)
await job_service.update_job_status(
job_id="test-job-123",
status="processing"
)
# Extract data from Lua script call
lua_mock = mock_redis.register_script.return_value
stored_data = _extract_lua_mapping(lua_mock)
# Verify updated_at is timezone-aware
updated_at_str = stored_data['updated_at']
updated_at = datetime.fromisoformat(updated_at_str.replace("Z", "+00:00"))
assert updated_at.tzinfo is not None
assert updated_at.tzinfo == UTC or updated_at.tzinfo.utcoffset(None) == timedelta(0)
@pytest.mark.asyncio
async def test_queue_service_creates_timezone_aware_datetimes(self, mock_redis):
"""Test QueueService.queue_pii_job stores timezone-aware timestamps."""
queue_service = QueueService(mock_redis)
await queue_service.queue_pii_job(
job_id="test-job-123",
s3_key="temp/test.pdf"
)
# Get the queued payload
call_args = mock_redis.lpush.call_args
import json
payload = json.loads(call_args[0][1])
# Verify created_at is timezone-aware
created_at_str = payload['created_at']
created_at = datetime.fromisoformat(created_at_str.replace("Z", "+00:00"))
assert created_at.tzinfo is not None
assert created_at.tzinfo == UTC or created_at.tzinfo.utcoffset(None) == timedelta(0)
class TestDatetimeParsing:
"""Test that all datetime parsing creates timezone-aware objects."""
@pytest.mark.asyncio
async def test_approval_service_parses_timezone_aware_datetimes(self, mock_redis, mock_s3):
"""Test ApprovalService.validate_approval_token parses timezone-aware expiration."""
job_service = JobService(mock_redis)
queue_service = QueueService(mock_redis)
approval_service = ApprovalService(
redis_client=mock_redis,
s3_client=mock_s3,
job_service=job_service,
queue_service=queue_service
)
# Mock job data with ISO format timestamp
future_time = datetime.now(UTC) + timedelta(hours=2)
mock_job_data = {
"job_id": "test-job-123",
"approval_token": "test-token-123",
"approval_expires_at": future_time.isoformat(),
"status": "awaiting_approval",
"s3_key": "temp/test.pdf"
}
# Mock Redis keys and job retrieval
mock_redis.keys.return_value = ["eq-pdf:job:test-job-123"]
job_service.get_job = AsyncMock(return_value=mock_job_data)
# Validate token
result = await approval_service.validate_approval_token("test-token-123")
# Should successfully validate (not expired)
assert result is not None
assert result["job_id"] == "test-job-123"
@pytest.mark.asyncio
async def test_approval_service_handles_z_suffix_in_timestamps(self, mock_redis, mock_s3):
"""Test ApprovalService handles Z suffix in ISO timestamps."""
job_service = JobService(mock_redis)
queue_service = QueueService(mock_redis)
approval_service = ApprovalService(
redis_client=mock_redis,
s3_client=mock_s3,
job_service=job_service,
queue_service=queue_service
)
# Mock job data with Z suffix timestamp
future_time = datetime.now(UTC) + timedelta(hours=2)
mock_job_data = {
"job_id": "test-job-123",
"approval_token": "test-token-123",
"approval_expires_at": future_time.isoformat().replace("+00:00", "Z"),
"status": "awaiting_approval",
"s3_key": "temp/test.pdf"
}
mock_redis.keys.return_value = ["eq-pdf:job:test-job-123"]
job_service.get_job = AsyncMock(return_value=mock_job_data)
result = await approval_service.validate_approval_token("test-token-123")
# Should handle Z suffix correctly
assert result is not None
@pytest.mark.asyncio
async def test_approval_service_rejects_expired_tokens(self, mock_redis, mock_s3):
"""Test ApprovalService correctly identifies expired tokens."""
job_service = JobService(mock_redis)
queue_service = QueueService(mock_redis)
approval_service = ApprovalService(
redis_client=mock_redis,
s3_client=mock_s3,
job_service=job_service,
queue_service=queue_service
)
# Mock job data with expired timestamp
past_time = datetime.now(UTC) - timedelta(hours=1)
mock_job_data = {
"job_id": "test-job-123",
"approval_token": "test-token-123",
"approval_expires_at": past_time.isoformat(),
"status": "awaiting_approval",
"s3_key": "temp/test.pdf"
}
mock_redis.keys.return_value = ["eq-pdf:job:test-job-123"]
job_service.get_job = AsyncMock(return_value=mock_job_data)
result = await approval_service.validate_approval_token("test-token-123")
# Should return None for expired token
assert result is None
class TestDatetimeComparisons:
"""Test that datetime comparisons work correctly across services."""
@pytest.mark.asyncio
async def test_approval_expiration_comparison_with_utc_now(self, mock_redis, mock_s3):
"""Test approval expiration comparison uses consistent timezone-aware datetimes."""
job_service = JobService(mock_redis)
queue_service = QueueService(mock_redis)
approval_service = ApprovalService(
redis_client=mock_redis,
s3_client=mock_s3,
job_service=job_service,
queue_service=queue_service
)
# Create datetimes using the same method as the code
now = datetime.now(UTC)
future = now + timedelta(hours=1)
# Mock job with future expiration
mock_job_data = {
"job_id": "test-job-123",
"approval_token": "test-token-123",
"approval_expires_at": future.isoformat(),
"status": "awaiting_approval",
"s3_key": "temp/test.pdf"
}
mock_redis.keys.return_value = ["eq-pdf:job:test-job-123"]
job_service.get_job = AsyncMock(return_value=mock_job_data)
result = await approval_service.validate_approval_token("test-token-123")
# Should be valid (future expiration)
assert result is not None
@pytest.mark.asyncio
async def test_timeout_tracking_comparison(self, mock_redis):
"""Test timeout tracking uses consistent timestamp comparisons."""
queue_service = QueueService(mock_redis)
# Add job to timeout tracking with future expiration
future_time = datetime.now(UTC) + timedelta(hours=2)
await queue_service.add_to_timeout_tracking(
job_id="test-job-123",
expires_at=future_time,
timeout_type="approval"
)
# Mock Redis response for get_expired_timeouts
mock_redis.zrangebyscore.return_value = [] # No expired jobs yet
# Get expired timeouts (should be empty since job expires in future)
expired = await queue_service.get_expired_timeouts("approval")
assert len(expired) == 0 # No expired jobs
class TestTimezoneEdgeCases:
"""Test edge cases for timezone handling."""
@pytest.mark.asyncio
async def test_utc_midnight_boundary(self, mock_redis):
"""Test datetime handling at UTC midnight boundary."""
job_service = JobService(mock_redis)
# Create job at UTC midnight
with patch('src.services.job_service.datetime') as mock_datetime:
midnight = datetime(2024, 1, 1, 0, 0, 0, tzinfo=UTC)
mock_datetime.now.return_value = midnight
mock_datetime.fromisoformat = datetime.fromisoformat
await job_service.create_job(
job_id="midnight-job",
s3_key="temp/midnight.pdf"
)
lua_mock = mock_redis.register_script.return_value
stored_data = _extract_lua_mapping(lua_mock)
created_at_str = stored_data['created_at']
# Should handle midnight correctly
created_at = datetime.fromisoformat(created_at_str.replace("Z", "+00:00"))
assert created_at.tzinfo is not None
assert created_at.hour == 0
assert created_at.minute == 0
@pytest.mark.asyncio
async def test_dst_transition_handling(self, mock_redis):
"""Test datetime handling during DST transitions (UTC is unaffected)."""
job_service = JobService(mock_redis)
# DST transition date (March 2024) - UTC is always consistent
dst_date = datetime(2024, 3, 10, 2, 0, 0, tzinfo=UTC)
with patch('src.services.job_service.datetime') as mock_datetime:
mock_datetime.now.return_value = dst_date
mock_datetime.fromisoformat = datetime.fromisoformat
await job_service.create_job(
job_id="dst-job",
s3_key="temp/dst.pdf"
)
lua_mock = mock_redis.register_script.return_value
stored_data = _extract_lua_mapping(lua_mock)
created_at_str = stored_data['created_at']
# UTC should be unaffected by DST
created_at = datetime.fromisoformat(created_at_str.replace("Z", "+00:00"))
assert created_at.tzinfo is not None
assert created_at == dst_date
@pytest.mark.asyncio
async def test_year_boundary_handling(self, mock_redis):
"""Test datetime handling at year boundaries."""
job_service = JobService(mock_redis)
# New Year's Eve at 23:59:59 UTC
year_end = datetime(2024, 12, 31, 23, 59, 59, tzinfo=UTC)
with patch('src.services.job_service.datetime') as mock_datetime:
mock_datetime.now.return_value = year_end
mock_datetime.fromisoformat = datetime.fromisoformat
await job_service.create_job(
job_id="year-end-job",
s3_key="temp/year-end.pdf"
)
lua_mock = mock_redis.register_script.return_value
stored_data = _extract_lua_mapping(lua_mock)
created_at_str = stored_data['created_at']
created_at = datetime.fromisoformat(created_at_str.replace("Z", "+00:00"))
assert created_at.tzinfo is not None
assert created_at.year == 2024
assert created_at.month == 12
assert created_at.day == 31
class TestJobLifecycleTimezoneConsistency:
"""Test timezone consistency throughout entire job lifecycle."""
@pytest.mark.asyncio
async def test_job_creation_to_retrieval_timezone_consistency(self, mock_redis):
"""Test that job timestamps remain timezone-aware from creation to retrieval."""
job_service = JobService(mock_redis)
# Create job
await job_service.create_job(
job_id="lifecycle-job",
s3_key="temp/lifecycle.pdf",
status="pii_scanning"
)
# Extract stored data from Lua script
lua_mock = mock_redis.register_script.return_value
stored_data = _extract_lua_mapping(lua_mock)
mock_redis.hgetall.return_value = stored_data
# Retrieve job
job_data = await job_service.get_job("lifecycle-job")
# Verify timezone awareness is preserved
created_at_str = job_data['created_at']
created_at = datetime.fromisoformat(created_at_str.replace("Z", "+00:00"))
assert created_at.tzinfo is not None
@pytest.mark.asyncio
async def test_job_update_preserves_timezone_awareness(self, mock_redis):
"""Test that job updates maintain timezone-aware timestamps."""
job_service = JobService(mock_redis)
# Update job
await job_service.update_job_status(
job_id="update-job",
status="processing"
)
# Extract data from Lua script call
lua_mock = mock_redis.register_script.return_value
stored_data = _extract_lua_mapping(lua_mock)
updated_at_str = stored_data['updated_at']
updated_at = datetime.fromisoformat(updated_at_str.replace("Z", "+00:00"))
assert updated_at.tzinfo is not None
@pytest.mark.asyncio
async def test_approval_expiration_workflow_timezone_consistency(self, mock_redis, mock_s3):
"""Test approval expiration workflow maintains timezone consistency."""
job_service = JobService(mock_redis)
queue_service = QueueService(mock_redis)
approval_service = ApprovalService(
redis_client=mock_redis,
s3_client=mock_s3,
job_service=job_service,
queue_service=queue_service
)
# Create approval with 4-hour expiration
now = datetime.now(UTC)
expires_at = now + timedelta(hours=4)
mock_job_data = {
"job_id": "approval-job",
"approval_token": "token-123",
"approval_expires_at": expires_at.isoformat(),
"status": "awaiting_approval",
"s3_key": "temp/approval.pdf"
}
mock_redis.keys.return_value = ["eq-pdf:job:approval-job"]
job_service.get_job = AsyncMock(return_value=mock_job_data)
# Validate token (should be valid)
result = await approval_service.validate_approval_token("token-123")
assert result is not None
# Simulate time passing (5 hours)
future_now = now + timedelta(hours=5)
mock_job_data["approval_expires_at"] = expires_at.isoformat()
with patch('src.services.approval_service.datetime') as mock_datetime:
mock_datetime.now.return_value = future_now
mock_datetime.fromisoformat = datetime.fromisoformat
# Validate token again (should be expired)
result = await approval_service.validate_approval_token("token-123")
assert result is None # Expired
# Pytest fixtures
@pytest.fixture
def mock_redis():
"""Mock Redis client for testing."""
redis = AsyncMock()
redis.hset = AsyncMock()
redis.hgetall = AsyncMock()
redis.lpush = AsyncMock()
redis.keys = AsyncMock()
redis.zadd = AsyncMock()
redis.zrangebyscore = AsyncMock()
redis.zrem = AsyncMock()
# register_script is a SYNC method returning a callable Script object.
# AsyncMock would return a coroutine (not callable), so use MagicMock.
redis.register_script = MagicMock(return_value=AsyncMock())
return redis
@pytest.fixture
def mock_s3():
"""Mock S3 client for testing."""
return AsyncMock()