1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388"""Edge case tests for PII detection accuracy and validation."""
from unittest.mock import Mock, patch
import pytest
from src.services.pii_analyzer import PIIAnalyzer
from tests.e2e.edge_cases.helpers import (
generate_valid_academic_content,
)
@pytest.fixture
def pii_analyzer():
"""Create PII analyzer with default confidence threshold."""
# Mock the spaCy model loading to avoid downloading in tests
with patch('presidio_analyzer.nlp_engine.NlpEngineProvider') as mock_provider:
mock_engine = Mock()
mock_provider.return_value.create_engine.return_value = mock_engine
with patch('presidio_analyzer.AnalyzerEngine') as mock_analyzer:
# Use default threshold from settings (0.85)
analyzer = PIIAnalyzer()
# Set up the mock to return expected structure
analyzer.analyzer = mock_analyzer.return_value
yield analyzer
class TestPIIDetectionAccuracy:
"""Tests for PII detection accuracy with various input patterns."""
@pytest.mark.asyncio
async def test_ssn_detection_valid_formats(self, pii_analyzer):
"""Test SSN detection with various valid formats."""
test_cases = [
("My SSN is 123-45-6789", True, "Standard format with dashes"),
("SSN: 123456789", True, "No dashes"),
("Social Security Number 123-45-6789 required", True, "In sentence"),
]
for text, should_detect, description in test_cases:
# Mock Presidio response
if should_detect:
mock_result = Mock()
mock_result.entity_type = "US_SSN"
mock_result.start = text.find("123")
mock_result.end = mock_result.start + 11
mock_result.score = 0.95
pii_analyzer.analyzer.analyze.return_value = [mock_result]
else:
pii_analyzer.analyzer.analyze.return_value = []
findings = pii_analyzer.analyze_text(text)
if should_detect:
assert len(findings) > 0, f"Failed to detect SSN: {description}"
assert findings[0].entity_type == "US_SSN"
assert findings[0].score >= 0.85
else:
assert len(findings) == 0, f"False positive: {description}"
@pytest.mark.asyncio
async def test_ssn_detection_invalid_formats(self, pii_analyzer):
"""Test that invalid SSN patterns are not detected."""
invalid_ssns = [
"111-11-1111", # Invalid sequence
"000-00-0000", # All zeros
"123-45-678", # Too short
"123-456-7890", # Wrong format
"999-99-9999", # Out of valid range
]
for ssn in invalid_ssns:
text = f"Test SSN: {ssn}"
# Mock no detection for invalid SSNs
pii_analyzer.analyzer.analyze.return_value = []
findings = pii_analyzer.analyze_text(text)
# Should not detect invalid SSNs
assert len([f for f in findings if f.entity_type == "US_SSN"]) == 0
@pytest.mark.asyncio
async def test_email_detection_accuracy(self, pii_analyzer):
"""Test email detection with various formats."""
test_cases = [
("Contact: user@example.com", True, "Simple email"),
("Email: first.last@company.co.uk", True, "Subdomain and TLD"),
("user+tag@example.com", True, "Plus addressing"),
("user_name@example.com", True, "Underscore"),
("user@", False, "Incomplete email"),
("@example.com", False, "Missing user"),
("not-an-email", False, "No @ symbol"),
]
for text, should_detect, description in test_cases:
if should_detect:
# Find email in text
at_pos = text.find("@")
start = max(0, text[:at_pos].rfind(" ") + 1)
end = text.find(" ", at_pos) if " " in text[at_pos:] else len(text)
mock_result = Mock()
mock_result.entity_type = "EMAIL_ADDRESS"
mock_result.start = start
mock_result.end = end
mock_result.score = 0.90
pii_analyzer.analyzer.analyze.return_value = [mock_result]
else:
pii_analyzer.analyzer.analyze.return_value = []
findings = pii_analyzer.analyze_text(text)
if should_detect:
assert len(findings) > 0, f"Failed to detect: {description}"
assert any(f.entity_type == "EMAIL_ADDRESS" for f in findings)
else:
email_findings = [f for f in findings if f.entity_type == "EMAIL_ADDRESS"]
assert len(email_findings) == 0, f"False positive: {description}"
@pytest.mark.asyncio
async def test_phone_number_detection(self, pii_analyzer):
"""Test phone number detection with various formats."""
test_cases = [
("Call (555) 123-4567", True, "Standard US format"),
("Phone: 555-123-4567", True, "Dashes only"),
("Contact 5551234567", True, "No separators"),
("Tel: +1 555-123-4567", True, "International format"),
("000-000-0000", False, "All zeros"),
("123-456", False, "Too short"),
]
for text, should_detect, description in test_cases:
if should_detect:
# Find phone number position
import re
match = re.search(r'[\d\(\)\-\+\s]+', text)
if match:
mock_result = Mock()
mock_result.entity_type = "PHONE_NUMBER"
mock_result.start = match.start()
mock_result.end = match.end()
mock_result.score = 0.85
pii_analyzer.analyzer.analyze.return_value = [mock_result]
else:
pii_analyzer.analyzer.analyze.return_value = []
findings = pii_analyzer.analyze_text(text)
if should_detect:
assert len(findings) > 0, f"Failed to detect: {description}"
assert any(f.entity_type == "PHONE_NUMBER" for f in findings)
@pytest.mark.asyncio
async def test_person_name_not_detected(self, pii_analyzer):
"""Test that person names are NOT detected (PERSON entity disabled)."""
test_cases = [
"Student John Smith submitted",
"Contact Dr. Jane Doe",
"Professor Maria Garcia Lopez",
"Dylan Isaac Software Engineer",
]
for text in test_cases:
# PERSON entity type is disabled, should not detect
pii_analyzer.analyzer.analyze.return_value = []
findings = pii_analyzer.analyze_text(text)
# Should not detect PERSON entities (disabled to reduce false positives)
person_findings = [f for f in findings if f.entity_type == "PERSON"]
assert len(person_findings) == 0, f"Should not detect PERSON in: {text}"
@pytest.mark.asyncio
async def test_false_positive_prevention(self, pii_analyzer):
"""Test that academic patterns are not flagged as PII."""
academic_text = generate_valid_academic_content()
# Mock no PII detection in academic content
pii_analyzer.analyzer.analyze.return_value = []
findings = pii_analyzer.analyze_text(academic_text)
# Should not detect PII in clean academic content
assert len(findings) == 0
@pytest.mark.asyncio
async def test_course_codes_not_detected_as_ssn(self, pii_analyzer):
"""Test that course codes like CS-123-45-6789 are not flagged as SSN."""
course_text = "Course CS-123-45-6789 and MATH-111-22-3333"
# Mock no detection of course codes as SSN
pii_analyzer.analyzer.analyze.return_value = []
findings = pii_analyzer.analyze_text(course_text)
# Should not flag course codes
ssn_findings = [f for f in findings if f.entity_type == "US_SSN"]
assert len(ssn_findings) == 0
@pytest.mark.asyncio
async def test_location_not_detected(self, pii_analyzer):
"""Test that locations are NOT detected (LOCATION entity disabled)."""
test_cases = [
"Office at 123 Main Street, Chicago IL",
"Located in Chicago",
"Building 5, Room 201",
"San Francisco Bay Area",
]
for text in test_cases:
# LOCATION entity type is disabled, should not detect
pii_analyzer.analyzer.analyze.return_value = []
findings = pii_analyzer.analyze_text(text)
# Should not detect LOCATION entities (disabled to reduce false positives)
location_findings = [f for f in findings if f.entity_type == "LOCATION"]
assert len(location_findings) == 0, f"Should not detect LOCATION in: {text}"
@pytest.mark.asyncio
async def test_credit_card_detection(self, pii_analyzer):
"""Test credit card number detection."""
test_cases = [
("Card: 4532-1234-5678-9010", True, "Visa format"),
("Payment 5425233430109903", True, "Mastercard"),
("1234-5678-9012-3456", False, "Invalid checksum"),
]
for text, should_detect, description in test_cases:
if should_detect:
mock_result = Mock()
mock_result.entity_type = "CREDIT_CARD"
mock_result.start = 6
mock_result.end = 25
mock_result.score = 0.95
pii_analyzer.analyzer.analyze.return_value = [mock_result]
else:
pii_analyzer.analyzer.analyze.return_value = []
findings = pii_analyzer.analyze_text(text)
if should_detect:
assert len(findings) > 0, f"Failed to detect: {description}"
assert any(f.entity_type == "CREDIT_CARD" for f in findings)
@pytest.mark.asyncio
async def test_confidence_threshold_filtering(self, pii_analyzer):
"""Test that low confidence findings are filtered out."""
text = "Email: test@example.com"
# Mock low confidence finding (below 0.85 threshold)
mock_result = Mock()
mock_result.entity_type = "EMAIL_ADDRESS"
mock_result.start = 7
mock_result.end = 23
mock_result.score = 0.6 # Below 0.85 threshold
pii_analyzer.analyzer.analyze.return_value = [mock_result]
findings = pii_analyzer.analyze_text(text)
# Should be filtered out due to low confidence
assert len(findings) == 0
@pytest.mark.asyncio
async def test_multiple_pii_types_in_same_text(self, pii_analyzer):
"""Test detection of multiple PII types in one text."""
text = "Contact john.doe@example.com or 555-123-4567. SSN: 123-45-6789"
# Mock multiple findings (only pattern-based entities)
mock_results = [
Mock(entity_type="EMAIL_ADDRESS", start=8, end=29, score=0.95),
Mock(entity_type="PHONE_NUMBER", start=33, end=45, score=0.90),
Mock(entity_type="US_SSN", start=52, end=63, score=0.95),
]
pii_analyzer.analyzer.analyze.return_value = mock_results
findings = pii_analyzer.analyze_text(text)
# Should detect all three types
assert len(findings) == 3
entity_types = {f.entity_type for f in findings}
assert "EMAIL_ADDRESS" in entity_types
assert "PHONE_NUMBER" in entity_types
assert "US_SSN" in entity_types
@pytest.mark.asyncio
async def test_international_phone_formats(self, pii_analyzer):
"""Test detection of international phone number formats."""
test_cases = [
"+1 312-555-1234", # US
"+44 20 7123 4567", # UK
"+81 3-1234-5678", # Japan
"+86 10-1234-5678", # China
]
for phone in test_cases:
text = f"Phone: {phone}"
mock_result = Mock()
mock_result.entity_type = "PHONE_NUMBER"
mock_result.start = 7
mock_result.end = len(text)
mock_result.score = 0.85
pii_analyzer.analyzer.analyze.return_value = [mock_result]
findings = pii_analyzer.analyze_text(text)
# Should detect international formats
assert len(findings) > 0
assert findings[0].entity_type == "PHONE_NUMBER"
@pytest.mark.asyncio
async def test_date_time_not_detected(self, pii_analyzer):
"""Test that dates are NOT detected (DATE_TIME entity disabled)."""
test_cases = [
"Due date: January 15, 2024",
"Born on 03/15/1995",
"Meeting at 2:00 PM",
"The year 2024",
"September 2019 - Present",
]
for text in test_cases:
# DATE_TIME entity type is disabled, should not detect
pii_analyzer.analyzer.analyze.return_value = []
findings = pii_analyzer.analyze_text(text)
# Should not detect DATE_TIME entities (disabled to reduce false positives)
date_findings = [f for f in findings if f.entity_type == "DATE_TIME"]
assert len(date_findings) == 0, f"Should not detect DATE_TIME in: {text}"
@pytest.mark.asyncio
async def test_empty_text_handling(self, pii_analyzer):
"""Test handling of empty text input."""
pii_analyzer.analyzer.analyze.return_value = []
findings = pii_analyzer.analyze_text("")
assert len(findings) == 0
@pytest.mark.asyncio
async def test_very_long_text_handling(self, pii_analyzer):
"""Test handling of very long text content."""
# Create 10,000 word text
long_text = " ".join(["word"] * 10000)
pii_analyzer.analyzer.analyze.return_value = []
# Should handle without error
findings = pii_analyzer.analyze_text(long_text)
assert isinstance(findings, list)
@pytest.mark.asyncio
async def test_special_characters_in_text(self, pii_analyzer):
"""Test PII detection with special characters."""
text = "Email: user@example.com\n\nPhone: (555) 123-4567\r\nSSN: 123-45-6789"
mock_results = [
Mock(entity_type="EMAIL_ADDRESS", start=7, end=23, score=0.95),
Mock(entity_type="PHONE_NUMBER", start=32, end=46, score=0.90),
Mock(entity_type="US_SSN", start=53, end=64, score=0.95),
]
pii_analyzer.analyzer.analyze.return_value = mock_results
findings = pii_analyzer.analyze_text(text)
# Should detect PII despite special characters
assert len(findings) == 3
@pytest.mark.asyncio
async def test_obfuscated_pii_patterns(self, pii_analyzer):
"""Test detection of obfuscated PII patterns."""
test_cases = [
"SSN: XXX-XX-6789", # Partially masked
"Email: j***@example.com", # Masked email
"Phone: (555) XXX-XXXX", # Masked phone
]
for text in test_cases:
# Obfuscated patterns may or may not be detected
pii_analyzer.analyzer.analyze.return_value = []
findings = pii_analyzer.analyze_text(text)
# Just verify no errors occur
assert isinstance(findings, list)