Added Weibo and Trello, transforms improvements

CHANGELOG.md

@@ -1,6 +1,8 @@
 # Changelog
 
 ## [Unreleased]
+* non-json transforms
+* added Weibo and Trello
 * added username-from-email extractor
 * added Yelp
 

socid_extractor/cli.py

@@ -22,12 +22,15 @@ def get_site_response(url, cookies=None, headers={}):
 
 
 def run():
-    parser = argparse.ArgumentParser(description=f'Extract accounts\' identifiers from pages. {len(schemes)} sites (methods) are supported.')
+    parser = argparse.ArgumentParser(
+        description=f'Extract accounts\' identifiers from pages. {len(schemes)} sites (methods) are supported.',
+        prog='socid_extractor',
+    )
     parser.add_argument('--url', help='url to parse')
     parser.add_argument('--cookies', default='', help='cookies to make http requests with auth')
     parser.add_argument('-v', '--verbose', action='store_true', help='display verbose information')
     parser.add_argument('-d', '--debug', action='store_true', help='display debug information')
-    parser.add_argument('--file', action='store_true', help='load from file instead of URL')
+    parser.add_argument('--file', help='file to parse')
     parser.add_argument('--activation', type=str, help='use certain type of request activation')
 
     args = parser.parse_args()

socid_extractor/main.py

@@ -9,7 +9,7 @@
 
 HEADERS = {
     'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36',
-    "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9"
+    "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
 }
 
 PROCESS_ERRORS = (AttributeError, KeyError, IndexError, TypeError)
@@ -52,6 +52,31 @@ def mutate_url(url):
     return mutate_results
 
 
+def transform(scheme_data, extracted_data):
+    transforms = scheme_data.get('transforms', [])
+    if transforms:
+        for t in transforms:
+            logging.debug(t)
+            try:
+                extracted_data = t(extracted_data)
+            except PROCESS_ERRORS as e:
+                logging.debug(f'Transform error: {e}')
+                extracted_data = {}
+            logging.debug(extracted_data)
+    return extracted_data
+
+
+def map_fields(scheme_data, transformed_data):
+    values = {}
+    for name, get_field in scheme_data['fields'].items():
+        try:
+            value = get_field(transformed_data)
+            values[name] = str(value) if value not in (None, [], {}) else ''
+        except PROCESS_ERRORS as e:
+            logging.debug(f'Unable to extact field {name}: {e}')
+    return values
+
+
 def extract(page):
     for scheme_name, scheme_data in schemes.items():
         flags = scheme_data['flags']
@@ -80,21 +105,11 @@ def extract(page):
 
             if scheme_data.get('extract_json', False):
                 extracted = regexp_group.group(1)
-
                 logging.debug('Extracted: %s', extracted)
 
-                transforms = scheme_data.get('transforms', [])
-                if transforms:
-                    for t in transforms:
-                        logging.debug(t)
-                        try:
-                            extracted = t(extracted)
-                        except PROCESS_ERRORS as e:
-                            logging.debug(f'Transform error: {e}')
-                            extracted = {}
-                        logging.debug(extracted)
+                transformed = transform(scheme_data, extracted)
 
-                json_data = json.loads(extracted)
+                json_data = json.loads(transformed)
 
                 if json_data == {}:
                     logging.debug('Unabled to extract json!')
@@ -107,14 +122,17 @@ def extract(page):
                     with open('debug_extracted.json', 'w') as f:
                         f.write(loaded_json_str)
 
-                for name, get_field in scheme_data['fields'].items():
-                    try:
-                        value = get_field(json_data)
-                        values[name] = str(value) if value not in (None, [], {}) else ''
-                    except PROCESS_ERRORS as e:
-                        logging.debug(f'Unable to extact field {name}: {e}')
+                values = map_fields(scheme_data, json_data)
             else:
-                values = regexp_group.groupdict()
+                groupdict = regexp_group.groupdict()
+                if groupdict:
+                    values = groupdict
+                else:
+                    extracted = regexp_group.group(1)
+                    logging.debug('Extracted: %s', extracted)
+
+                    transformed_data = transform(scheme_data, extracted)
+                    values = map_fields(scheme_data, transformed_data)
 
         if use_html_parser:
             soup = bs(page, 'html.parser')

socid_extractor/schemes.py

@@ -1343,5 +1343,38 @@
             'location': lambda x: x.find('div', {'class': 'user-profile_info'}).find('h3').contents[0].split(' ', 1)[1],
             'image': lambda x: x.find('div', {'class': 'user-profile_avatar'}).find('img').get('src'),
         }
+    },
+    'Trello API': {
+        'flags': ['"aaId"', '"trophies":'],
+        'regex': r'^([\s\S]+)$',
+        'extract_json': True,
+        'fields': {
+            'id': lambda x: x['id'],
+            'username': lambda x: x['username'],
+            'fullname': lambda x: x['fullName'],
+            'email': lambda x: x['email'],
+            'image': lambda x: x['avatarUrl'] + '/170.png',
+            'bio': lambda x: x['bio'],
+            'type': lambda x: x['memberType'],
+            'gravatar_email_md5_hash': lambda x: x['gravatarHash'],
+            'is_verified': lambda x: x['confirmed'],
+        }
+    },
+    'Weibo': {
+        'flags': ['$CONFIG[\'timeweibo\']'],
+        'regex': r'\$CONFIG = {};[\r\n]([\s\S]+?)</script>',
+        'transforms': [
+            lambda x: re.split('[\r\n]', x),
+            lambda x: [r.split("'") for r in x if r],
+            lambda x: {r[1]: r[-2] for r in x},
+        ],
+        'fields': {
+            'weibo_id': lambda x: x['oid'],
+            'fullname': lambda x: x['onick'],
+            'nickname': lambda x: x['nick'],
+            'image': lambda x: x['avatar_large'],
+            'gender': lambda x: x['sex'],
+            'language': lambda x: x['lang'],
+        }
     }
 }

tests/reformat.sh

@@ -0,0 +1,2 @@
+#!/bin/bash
+sed -i '' -r 's/^([0-9a-zA-Z_]+): (.+)$/    assert info.get("\1") == "\2"/g' test_e2e.py

tests/test_e2e.py

@@ -972,3 +972,22 @@ def test_yelp_userid():
     assert info.get('fullname') == 'Dima "ZOMG" M.'
     assert info.get('location') == 'Brooklyn, NY'
     assert info.get('image') == 'https://s3-media0.fl.yelpcdn.com/photo/bGiNMDL6FZAtPpMfljRGtg/ls.jpg'
+
+
+def test_trello():
+    info = extract(parse('https://trello.com/1/Members/xFubuki')[0])
+
+    assert info.get("id") == "5e78cae55d711a6382e239c1"
+    assert info.get("username") == "xfubuki"
+    assert info.get("fullname") == "xFubuki"
+    assert info.get("image") == "https://trello-members.s3.amazonaws.com/5e78cae55d711a6382e239c1/d9c5264e657de6175f14a9067126873f/170.png"
+    assert info.get("type") == "normal"
+    assert info.get("is_verified") == "True"
+
+
+def test_weibo():
+    headers = {"cookie": "SUB=_2AkMXyuc_f8NxqwJRmP8SyWPrbo13zAvEieKhlhbkJRMxHRl-123", "cache-control": "no-cache"}
+    info = extract(parse('https://weibo.com/clairekuo?is_all=1', headers=headers, timeout=10)[0])
+
+    assert info.get("weibo_id") == "1733299783"
+    assert info.get("fullname") == "郭靜Claire"