217 lines
11 KiB
Python
217 lines
11 KiB
Python
import json
|
|
import unittest
|
|
from unittest.mock import Mock, patch
|
|
from urllib.parse import parse_qs, urlsplit
|
|
|
|
from .platform.douyin import BrowserResponse, DouyinBrowser, DouyinError
|
|
from .platform.notice_polling import NoticePollingSession, SubscriptionManager
|
|
|
|
NOW = '2026-10-06T10:00:00+00:00'
|
|
BOUNDARY = '2026-10-06T09:00:00+00:00'
|
|
|
|
|
|
def notice(nid='9007199254740993', timestamp=1791281100, uid='123', kind='follow'):
|
|
return {'nid': int(nid), 'user_id': int(uid), 'create_time': timestamp,
|
|
kind: {'from_user': [{'uid': 456}]}}
|
|
|
|
|
|
def page(items=(), more=0, cursor=0, status=0):
|
|
return BrowserResponse(200, json.dumps({'status_code': status, 'notice_list_v2': list(items),
|
|
'has_more': more, 'max_time': cursor}), False)
|
|
|
|
|
|
class NoticePollingTests(unittest.TestCase):
|
|
def browser(self, response=None):
|
|
browser = Mock()
|
|
browser.identity.return_value = {'uid': '123', 'platform_now': NOW}
|
|
browser.get.return_value = response or page()
|
|
browser._notification_browser.return_value = browser
|
|
return browser
|
|
|
|
def session(self, browser=None, checkpoints=None):
|
|
return NoticePollingSession(browser or self.browser(), 'safe', '123', BOUNDARY, checkpoints)
|
|
|
|
def test_initial_scan_backfills_history_and_reports_boundary_only_after_success(self):
|
|
browser = self.browser(page([notice(timestamp=1)]))
|
|
session = NoticePollingSession(browser, 'safe', '123')
|
|
self.assertEqual(session.pending(), [])
|
|
result = session.poll(100, 0)
|
|
self.assertEqual([d['kind'] for d in result], ['baseline', 'notice', 'checkpoint'])
|
|
self.assertTrue(result[1]['baseline'])
|
|
self.assertEqual(result[1]['notice']['platform_event_at'], '1970-01-01T00:00:01+00:00')
|
|
self.assertEqual(result[0]['boundary_at'], NOW)
|
|
for call in browser.get.call_args_list:
|
|
params = parse_qs(urlsplit(call.args[1]).query)
|
|
self.assertEqual(params['is_mark_read'], ['0'])
|
|
self.assertEqual(params['min_time'], ['0'])
|
|
self.assertEqual(params['count'], ['50'])
|
|
self.assertEqual(set(result[-1]['checkpoints']), {'700', '960', '961'})
|
|
|
|
def test_big_ids_and_duplicates_survive_replay_until_checkpoint_ack(self):
|
|
session = self.session(self.browser(page([notice(), notice()])))
|
|
first = session.poll(100, 0)
|
|
self.assertEqual(first, session.poll(100, 0))
|
|
events = [d for d in first if d['kind'] == 'notice']
|
|
self.assertEqual(len(events), 1)
|
|
self.assertEqual(events[0]['notice']['event_key'], '9007199254740993')
|
|
self.assertEqual(session.checkpoints, {})
|
|
session.ack([d['delivery_id'] for d in first[:-1]])
|
|
self.assertEqual(session.checkpoints, {})
|
|
session.ack([first[-1]['delivery_id']])
|
|
self.assertEqual(session.checkpoints['960'], '9007199254740993')
|
|
# Re-read the checkpoint page for out-of-order visibility; DB deduplicates it.
|
|
self.assertEqual(session.poll(100, 0)[-1]['kind'], 'checkpoint')
|
|
|
|
def test_incremental_scan_reads_entire_boundary_page_and_out_of_order_notices(self):
|
|
browser = self.browser()
|
|
def fetch(alias, url):
|
|
cursor = parse_qs(urlsplit(url).query)['max_time'][0]
|
|
if cursor == '0':
|
|
return page([notice('21', 1791281300)], 1, 1791281200)
|
|
return page([notice('20', 1791281200), notice('19', 1791281190)], 1, 1791281100)
|
|
browser.get.side_effect = fetch
|
|
session = self.session(browser, {'700': '20', '960': '20', '961': '20'})
|
|
session.last_full_scan_at = 1000
|
|
with patch('browser_gateway.platform.notice_polling.time.monotonic', return_value=1001):
|
|
result = session.poll(100, 0)
|
|
self.assertEqual({d['notice']['event_key'] for d in result if d['kind'] == 'notice'}, {'21', '20', '19'})
|
|
self.assertEqual(browser.get.call_count, 6)
|
|
|
|
def test_failure_never_advances_checkpoint_or_returns_partial_scan(self):
|
|
cases = [page(status=8), page([notice(uid='999')]), page([dict(notice(), nid=0)]),
|
|
page([dict(notice(), create_time=None)]), page([], 1, 0),
|
|
page([], 2), BrowserResponse(503, '{}', False),
|
|
BrowserResponse(200, 'not json', False), page([dict(notice(), follow='bad')])]
|
|
for bad in cases:
|
|
with self.subTest(body=bad.body):
|
|
session = self.session(self.browser(bad))
|
|
with self.assertRaises(DouyinError):
|
|
session.poll(100, 0)
|
|
self.assertEqual(session.pending(), [])
|
|
self.assertEqual(session.checkpoints, {})
|
|
|
|
def test_cursor_loops_fail_visibly(self):
|
|
session = self.session(self.browser(page([notice()], 1, 100)))
|
|
with self.assertRaisesRegex(DouyinError, 'cursor'):
|
|
session.poll(100, 0)
|
|
self.assertEqual(session.pending(), [])
|
|
|
|
def test_all_supported_interactions_have_distinct_types_and_accurate_actor(self):
|
|
raws = [notice(str(100 + i), kind=kind) for i, kind in enumerate(('follow','comment','digg','share'))]
|
|
result = self.session(self.browser(page(raws))).poll(100, 0)
|
|
events = [d['notice'] for d in result if d['kind'] == 'notice']
|
|
self.assertEqual({e['event_type'] for e in events}, {'follow','comment','like','repost'})
|
|
self.assertEqual({e['interactor_uid'] for e in events}, {'456'})
|
|
self.assertEqual(len(events), 4)
|
|
|
|
def test_restart_backfills_before_the_saved_head_and_before_enable_time(self):
|
|
browser = self.browser()
|
|
def fetch(alias, url):
|
|
cursor = parse_qs(urlsplit(url).query)['max_time'][0]
|
|
if cursor == '0':
|
|
return page([notice('21')], 1, 100)
|
|
return page([notice('20', 1)], 0)
|
|
browser.get.side_effect = fetch
|
|
session = self.session(browser, {'700':'21', '960':'21', '961':'21'})
|
|
deliveries = session.poll(100, 0)
|
|
events = [d['notice'] for d in deliveries if d['kind'] == 'notice']
|
|
self.assertEqual({e['event_key'] for e in events}, {'21','20'})
|
|
self.assertTrue(next(d for d in deliveries if d.get('notice',{}).get('event_key')=='20')['baseline'])
|
|
|
|
def test_periodic_full_reconciliation_recovers_old_holes_behind_the_head(self):
|
|
browser = self.browser()
|
|
missing_visible = False
|
|
def fetch(alias, url):
|
|
cursor = parse_qs(urlsplit(url).query)['max_time'][0]
|
|
if cursor == '0':
|
|
return page([notice('21')], 1, 100)
|
|
return page([notice('20', 1)] if missing_visible else [], 0)
|
|
browser.get.side_effect = fetch
|
|
session = self.session(browser)
|
|
with patch('browser_gateway.platform.notice_polling.time.monotonic', return_value=1000):
|
|
first = session.poll(100,0)
|
|
session.ack([d['delivery_id'] for d in first])
|
|
missing_visible = True
|
|
with patch('browser_gateway.platform.notice_polling.time.monotonic', return_value=1001):
|
|
incremental = session.poll(100,0)
|
|
self.assertNotIn('20', {d['notice']['event_key'] for d in incremental if d['kind']=='notice'})
|
|
session.ack([d['delivery_id'] for d in incremental])
|
|
with patch('browser_gateway.platform.notice_polling.time.monotonic', return_value=1301):
|
|
complete = session.poll(100,0)
|
|
self.assertIn('20', {d['notice']['event_key'] for d in complete if d['kind']=='notice'})
|
|
|
|
def test_history_scan_is_not_truncated_at_one_hundred_pages(self):
|
|
browser = self.browser()
|
|
def fetch(alias,url):
|
|
cursor = int(parse_qs(urlsplit(url).query)['max_time'][0])
|
|
current = cursor or 2000
|
|
return page([notice(str(current),1)], int(current>1899), current-1)
|
|
browser.get.side_effect = fetch
|
|
events = [d for d in self.session(browser).poll(1000,0) if d['kind']=='notice']
|
|
self.assertEqual(len(events), 102)
|
|
self.assertGreater(browser.get.call_count,300)
|
|
|
|
def test_private_messages_and_unrecognized_categories_are_not_aggregated(self):
|
|
session = self.session(self.browser(page([notice(kind='dm'), notice('2', kind='favorite')])))
|
|
result = session.poll(100, 0)
|
|
self.assertEqual([d['kind'] for d in result], ['checkpoint'])
|
|
self.assertEqual(result[-1]['ignored_types'], {'dm': 3, 'favorite': 3})
|
|
|
|
def test_checkpoint_missing_is_a_visible_gap_not_silent_continuity(self):
|
|
session = self.session(self.browser(page([notice()])), {'960': '404'})
|
|
result = session.poll(100, 0)
|
|
self.assertEqual(result[-1]['kind'], 'checkpoint')
|
|
self.assertIn('960', result[-1]['reason'])
|
|
|
|
def test_wrong_uid_and_invalid_restart_checkpoint_are_rejected(self):
|
|
browser = self.browser()
|
|
browser.identity.side_effect = DouyinError('identity mismatch')
|
|
with self.assertRaisesRegex(DouyinError, 'identity'):
|
|
self.session(browser)
|
|
for boundary, checkpoints in [('bad', {}), (BOUNDARY, {'bad': '1'}), (BOUNDARY, {'960': 'bad'}), (None, {'960': '1'})]:
|
|
with self.subTest(boundary=boundary, checkpoints=checkpoints), self.assertRaises(DouyinError):
|
|
NoticePollingSession(self.browser(), 'safe', '123', boundary, checkpoints)
|
|
|
|
@patch('browser_gateway.platform.notice_polling.NoticePollingSession.start')
|
|
def test_stop_prevents_further_reads_and_manager_restarts_from_persisted_state(self, start):
|
|
browser = self.browser()
|
|
manager = SubscriptionManager(browser)
|
|
self.assertTrue(manager.start('safe', '123', BOUNDARY, {'960': '9'})['connected'])
|
|
browser._notification_browser.assert_called_once_with('safe')
|
|
old = manager._items['safe']
|
|
manager.start('safe', '123', BOUNDARY, {'960': '10'})
|
|
with self.assertRaisesRegex(DouyinError, 'stopped'):
|
|
old.poll(100, 0)
|
|
self.assertEqual(manager._items['safe'].checkpoints, {'960': '10'})
|
|
manager.poll('safe', 100, 0)
|
|
manager.ack('safe', [d['delivery_id'] for d in manager._items['safe'].pending()])
|
|
manager.close()
|
|
manager.stop('safe')
|
|
with self.assertRaises(DouyinError):
|
|
manager.poll('safe', 100, 0)
|
|
with self.assertRaises(DouyinError):
|
|
manager.ack('safe', [])
|
|
|
|
def test_notifications_pin_one_page_when_multiple_tabs_exist(self):
|
|
browser = DouyinBrowser(lambda _: 'http://127.0.0.1:19000')
|
|
browser._target_list = Mock(return_value=(None, 19000, [
|
|
{'type':'page', 'id':'profile', 'url':'https://www.douyin.com/user/other'},
|
|
{'type':'page', 'id':'chat', 'url':'https://www.douyin.com/chat'},
|
|
{'type':'page', 'id':'self', 'url':'https://www.douyin.com/user/self'},
|
|
]))
|
|
self.assertEqual(browser._notification_browser('safe').target_id, 'self')
|
|
browser._target_list.return_value = (None, 19000, [{'type':'page','id':'external','url':'https://example.com'}])
|
|
with self.assertRaisesRegex(DouyinError, 'notification page is unavailable'):
|
|
browser._notification_browser('safe')
|
|
|
|
def test_changed_identity_after_start_is_reported(self):
|
|
browser = self.browser()
|
|
session = self.session(browser)
|
|
browser.identity.side_effect = DouyinError('identity changed')
|
|
with self.assertRaisesRegex(DouyinError, 'identity'):
|
|
session.poll(100, 0)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|