diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index d7415125df84d..d131eb83e9adf 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -256,7 +256,7 @@ - name: Faker sourceDefinitionId: dfd88b22-b603-4c3d-aad7-3701784586b1 dockerRepository: airbyte/source-faker - dockerImageTag: 0.1.1 + dockerImageTag: 0.1.3 documentationUrl: https://docs.airbyte.com/integrations/source-faker sourceType: api releaseStage: alpha diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 016600a1de701..df524fd1682a1 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -2133,7 +2133,7 @@ oauthFlowInitParameters: [] oauthFlowOutputParameters: - - "access_token" -- dockerImage: "airbyte/source-faker:0.1.1" +- dockerImage: "airbyte/source-faker:0.1.3" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/faker" connectionSpecification: @@ -2146,10 +2146,11 @@ properties: count: title: "Count" - description: "How many fake records should be generated" + description: "How many users should be generated in total. This setting\ + \ does not apply to the purchases or products stream." type: "integer" minimum: 1 - default: 100 + default: 1000 order: 0 seed: title: "Seed" @@ -2158,6 +2159,23 @@ type: "integer" default: -1 order: 1 + records_per_sync: + title: "Records Per Sync" + description: "How many fake records will be returned for each sync, for\ + \ each stream? By default, it will take 2 syncs to create the requested\ + \ 1000 records." + type: "integer" + minimum: 1 + default: 500 + order: 2 + records_per_slice: + title: "Records Per Stream Slice" + description: "How many fake records will be in each page (stream slice),\ + \ before a state message is emitted?" + type: "integer" + minimum: 1 + default: 100 + order: 3 supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] diff --git a/airbyte-integrations/connectors/source-faker/Dockerfile b/airbyte-integrations/connectors/source-faker/Dockerfile index 88bc02432d8c9..95d7e31bbbcd4 100644 --- a/airbyte-integrations/connectors/source-faker/Dockerfile +++ b/airbyte-integrations/connectors/source-faker/Dockerfile @@ -34,5 +34,5 @@ COPY source_faker ./source_faker ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.1 +LABEL io.airbyte.version=0.1.3 LABEL io.airbyte.name=airbyte/source-faker diff --git a/airbyte-integrations/connectors/source-faker/integration_tests/expected_records.txt b/airbyte-integrations/connectors/source-faker/integration_tests/expected_records.txt index 96a752e658473..185bd299ad991 100644 --- a/airbyte-integrations/connectors/source-faker/integration_tests/expected_records.txt +++ b/airbyte-integrations/connectors/source-faker/integration_tests/expected_records.txt @@ -1,10 +1,10 @@ -{"stream": "Users", "data": {"job": "Musician", "company": "Williams-Sheppard", "ssn": "498-52-4970", "residence": "Unit 5938 Box 2421\nDPO AP 33335", "current_location": [52.958961, 143.143712], "blood_group": "B+", "website": ["http://www.rivera.com/", "http://grimes-green.net/", "http://www.larsen.com/"], "username": "leeashley", "name": "Gary Cross", "sex": "M", "address": "711 Golden Overpass\nWest Andreaville, MA 71317", "mail": "tamaramorrison@hotmail.com", "id": 0, "created_at": "1976-03-27T12:40:22", "updated_at": "2000-04-13T06:17:38"}, "emitted_at": 1649892395000} -{"stream": "Users", "data": {"job": "Therapist, occupational", "company": "Stewart-Allen", "ssn": "189-25-3060", "residence": "1122 Megan Squares Suite 848\nPort Jason, OR 55475", "current_location": [65.610695, -32.24732], "blood_group": "O-", "website": ["https://www.salazar-tucker.com/", "http://www.dennis.com/", "https://www.simmons-brown.com/", "http://www.walters.com/"], "username": "myersmitchell", "name": "Chelsea Greer", "sex": "F", "address": "Unit 0903 Box 2173\nDPO AP 08507", "mail": "stephenschristine@yahoo.com", "id": 1, "created_at": "1974-11-24T20:47:03", "updated_at": "1985-01-10T17:47:40"}, "emitted_at": 1649892395000} -{"stream": "Users", "data": {"job": "Illustrator", "company": "Graham-Brown", "ssn": "479-06-9773", "residence": "45792 Tammy Centers Apt. 258\nDavidmouth, HI 02231", "current_location": [36.600983, 40.066283], "blood_group": "AB+", "website": ["https://castro.info/", "http://williams.info/", "https://howard.org/"], "username": "smithjames", "name": "Jasmine Perry", "sex": "F", "address": "76960 Savage Port\nBartonton, NV 08874", "mail": "jacqueline78@yahoo.com", "id": 2, "created_at": "2011-02-06T22:32:40", "updated_at": "2021-05-30T16:13:19"}, "emitted_at": 1649892395000} -{"stream": "Users", "data": {"job": "Academic librarian", "company": "Simpson LLC", "ssn": "858-60-0817", "residence": "81206 Stewart Forest Apt. 089\nEast Davidborough, ME 37198", "current_location": [-41.0853825, -126.215901], "blood_group": "O-", "website": ["http://www.sloan-marsh.com/", "https://www.meyer.com/", "https://www.williams.com/", "https://browning.org/"], "username": "monica23", "name": "Steven Bowman", "sex": "M", "address": "22455 Higgins Junction Apt. 042\nNew Keith, OH 17493", "mail": "danny30@yahoo.com", "id": 3, "created_at": "2005-08-05T11:36:52", "updated_at": "2019-08-02T23:04:14"}, "emitted_at": 1649892395000} -{"stream": "Users", "data": {"job": "Building services engineer", "company": "Powell-Murphy", "ssn": "425-14-1629", "residence": "5517 Holly Meadow Apt. 452\nLake Anne, SC 11894", "current_location": [-72.327639, -134.543372], "blood_group": "B-", "website": ["http://garcia.net/", "https://kramer-klein.com/"], "username": "ryanhoward", "name": "Daniel Duarte", "sex": "M", "address": "731 Sanders Fords\nPort Jasonberg, ID 60585", "mail": "jward@gmail.com", "id": 4, "created_at": "1980-02-06T13:15:11", "updated_at": "2003-03-15T20:31:32"}, "emitted_at": 1649892395000} -{"stream": "Users", "data": {"job": "Surveyor, rural practice", "company": "Little-Henderson", "ssn": "534-64-5284", "residence": "389 Alec Squares Suite 508\nPort Jonathan, FL 50177", "current_location": [66.4839605, -21.954682], "blood_group": "O-", "website": ["http://www.perez.com/"], "username": "mistymurray", "name": "Joan Atkins", "sex": "F", "address": "809 Erika Valley Apt. 634\nPetersenfort, WY 51431", "mail": "melissayates@hotmail.com", "id": 5, "created_at": "1982-05-20T20:41:25", "updated_at": "2020-05-29T07:50:41"}, "emitted_at": 1649892395000} -{"stream": "Users", "data": {"job": "Immigration officer", "company": "Hudson, Cook and Snyder", "ssn": "269-39-6686", "residence": "22345 Sheri Orchard Suite 279\nLake Hollystad, AZ 80687", "current_location": [35.165804, 27.598858], "blood_group": "O-", "website": ["http://smith.org/"], "username": "james71", "name": "Ashley Dunn", "sex": "F", "address": "519 Kramer Crossroad Suite 418\nNorth Kimberly, MN 99672", "mail": "stanleyclarke@gmail.com", "id": 6, "created_at": "1972-09-20T17:24:07", "updated_at": "2003-01-31T16:46:49"}, "emitted_at": 1649892395000} -{"stream": "Users", "data": {"job": "Freight forwarder", "company": "Duffy Ltd", "ssn": "583-74-3539", "residence": "412 Snow Manors Apt. 161\nSouth Kimtown, NV 57004", "current_location": [21.942823, -163.821807], "blood_group": "O-", "website": ["https://www.jones-howard.com/", "https://smith.com/", "http://www.smith.com/"], "username": "michelelopez", "name": "Melissa Cantu", "sex": "F", "address": "420 Michael Mountains Suite 485\nNew Victoria, ND 76634", "mail": "eric22@gmail.com", "id": 7, "created_at": "1986-07-11T18:55:27", "updated_at": "2004-08-04T16:06:58"}, "emitted_at": 1649892395000} -{"stream": "Users", "data": {"job": "Fast food restaurant manager", "company": "Carter Group", "ssn": "680-06-2167", "residence": "12648 Yang Divide Suite 451\nSouth Cynthia, NC 08084", "current_location": [11.047629, 39.379532], "blood_group": "A+", "website": ["https://watson.com/", "http://www.rodriguez-jacobs.com/", "https://saunders.com/", "http://giles-thomas.info/"], "username": "amontgomery", "name": "Christina Smith", "sex": "F", "address": "489 Roger Terrace\nDavisfort, IN 71770", "mail": "bramirez@gmail.com", "id": 8, "created_at": "1972-10-04T19:18:09", "updated_at": "2015-06-29T09:01:24"}, "emitted_at": 1649892395000} -{"stream": "Users", "data": {"job": "Academic librarian", "company": "Ross-Zamora", "ssn": "728-51-7285", "residence": "4391 Chad Greens Suite 851\nPort Frank, LA 37561", "current_location": [49.5419055, -107.833532], "blood_group": "B-", "website": ["https://osborne.com/"], "username": "qtaylor", "name": "Melissa James", "sex": "F", "address": "637 Neal Island Suite 074\nLake Tyler, RI 28775", "mail": "kellypeter@gmail.com", "id": 9, "created_at": "1974-07-10T19:02:25", "updated_at": "2005-11-07T09:09:06"}, "emitted_at": 1649892395000} +{"stream": "Users", "data": {"job": "Musician", "company": "Williams-Sheppard", "ssn": "498-52-4970", "residence": "Unit 5938 Box 2421\nDPO AP 33335", "current_location": [52.958961, 143.143712], "blood_group": "B+", "website": ["http://www.rivera.com/", "http://grimes-green.net/", "http://www.larsen.com/"], "username": "leeashley", "name": "Gary Cross", "sex": "M", "address": "711 Golden Overpass\nWest Andreaville, MA 71317", "mail": "tamaramorrison@hotmail.com", "id": 1, "created_at": "1976-03-27T12:40:22", "updated_at": "2000-04-13T06:17:38"}, "emitted_at": 1649892395000} +{"stream": "Users", "data": {"job": "Therapist, occupational", "company": "Stewart-Allen", "ssn": "189-25-3060", "residence": "1122 Megan Squares Suite 848\nPort Jason, OR 55475", "current_location": [65.610695, -32.24732], "blood_group": "O-", "website": ["https://www.salazar-tucker.com/", "http://www.dennis.com/", "https://www.simmons-brown.com/", "http://www.walters.com/"], "username": "myersmitchell", "name": "Chelsea Greer", "sex": "F", "address": "Unit 0903 Box 2173\nDPO AP 08507", "mail": "stephenschristine@yahoo.com", "id": 2, "created_at": "1974-11-24T20:47:03", "updated_at": "1985-01-10T17:47:40"}, "emitted_at": 1649892395000} +{"stream": "Users", "data": {"job": "Illustrator", "company": "Graham-Brown", "ssn": "479-06-9773", "residence": "45792 Tammy Centers Apt. 258\nDavidmouth, HI 02231", "current_location": [36.600983, 40.066283], "blood_group": "AB+", "website": ["https://castro.info/", "http://williams.info/", "https://howard.org/"], "username": "smithjames", "name": "Jasmine Perry", "sex": "F", "address": "76960 Savage Port\nBartonton, NV 08874", "mail": "jacqueline78@yahoo.com", "id": 3, "created_at": "2011-02-06T22:32:40", "updated_at": "2021-05-30T16:13:19"}, "emitted_at": 1649892395000} +{"stream": "Users", "data": {"job": "Academic librarian", "company": "Simpson LLC", "ssn": "858-60-0817", "residence": "81206 Stewart Forest Apt. 089\nEast Davidborough, ME 37198", "current_location": [-41.0853825, -126.215901], "blood_group": "O-", "website": ["http://www.sloan-marsh.com/", "https://www.meyer.com/", "https://www.williams.com/", "https://browning.org/"], "username": "monica23", "name": "Steven Bowman", "sex": "M", "address": "22455 Higgins Junction Apt. 042\nNew Keith, OH 17493", "mail": "danny30@yahoo.com", "id": 4, "created_at": "2005-08-05T11:36:52", "updated_at": "2019-08-02T23:04:14"}, "emitted_at": 1649892395000} +{"stream": "Users", "data": {"job": "Building services engineer", "company": "Powell-Murphy", "ssn": "425-14-1629", "residence": "5517 Holly Meadow Apt. 452\nLake Anne, SC 11894", "current_location": [-72.327639, -134.543372], "blood_group": "B-", "website": ["http://garcia.net/", "https://kramer-klein.com/"], "username": "ryanhoward", "name": "Daniel Duarte", "sex": "M", "address": "731 Sanders Fords\nPort Jasonberg, ID 60585", "mail": "jward@gmail.com", "id": 5, "created_at": "1980-02-06T13:15:11", "updated_at": "2003-03-15T20:31:32"}, "emitted_at": 1649892395000} +{"stream": "Users", "data": {"job": "Surveyor, rural practice", "company": "Little-Henderson", "ssn": "534-64-5284", "residence": "389 Alec Squares Suite 508\nPort Jonathan, FL 50177", "current_location": [66.4839605, -21.954682], "blood_group": "O-", "website": ["http://www.perez.com/"], "username": "mistymurray", "name": "Joan Atkins", "sex": "F", "address": "809 Erika Valley Apt. 634\nPetersenfort, WY 51431", "mail": "melissayates@hotmail.com", "id": 6, "created_at": "1982-05-20T20:41:25", "updated_at": "2020-05-29T07:50:41"}, "emitted_at": 1649892395000} +{"stream": "Users", "data": {"job": "Immigration officer", "company": "Hudson, Cook and Snyder", "ssn": "269-39-6686", "residence": "22345 Sheri Orchard Suite 279\nLake Hollystad, AZ 80687", "current_location": [35.165804, 27.598858], "blood_group": "O-", "website": ["http://smith.org/"], "username": "james71", "name": "Ashley Dunn", "sex": "F", "address": "519 Kramer Crossroad Suite 418\nNorth Kimberly, MN 99672", "mail": "stanleyclarke@gmail.com", "id": 7, "created_at": "1972-09-20T17:24:07", "updated_at": "2003-01-31T16:46:49"}, "emitted_at": 1649892395000} +{"stream": "Users", "data": {"job": "Freight forwarder", "company": "Duffy Ltd", "ssn": "583-74-3539", "residence": "412 Snow Manors Apt. 161\nSouth Kimtown, NV 57004", "current_location": [21.942823, -163.821807], "blood_group": "O-", "website": ["https://www.jones-howard.com/", "https://smith.com/", "http://www.smith.com/"], "username": "michelelopez", "name": "Melissa Cantu", "sex": "F", "address": "420 Michael Mountains Suite 485\nNew Victoria, ND 76634", "mail": "eric22@gmail.com", "id": 8, "created_at": "1986-07-11T18:55:27", "updated_at": "2004-08-04T16:06:58"}, "emitted_at": 1649892395000} +{"stream": "Users", "data": {"job": "Fast food restaurant manager", "company": "Carter Group", "ssn": "680-06-2167", "residence": "12648 Yang Divide Suite 451\nSouth Cynthia, NC 08084", "current_location": [11.047629, 39.379532], "blood_group": "A+", "website": ["https://watson.com/", "http://www.rodriguez-jacobs.com/", "https://saunders.com/", "http://giles-thomas.info/"], "username": "amontgomery", "name": "Christina Smith", "sex": "F", "address": "489 Roger Terrace\nDavisfort, IN 71770", "mail": "bramirez@gmail.com", "id": 9, "created_at": "1972-10-04T19:18:09", "updated_at": "2015-06-29T09:01:24"}, "emitted_at": 1649892395000} +{"stream": "Users", "data": {"job": "Academic librarian", "company": "Ross-Zamora", "ssn": "728-51-7285", "residence": "4391 Chad Greens Suite 851\nPort Frank, LA 37561", "current_location": [49.5419055, -107.833532], "blood_group": "B-", "website": ["https://osborne.com/"], "username": "qtaylor", "name": "Melissa James", "sex": "F", "address": "637 Neal Island Suite 074\nLake Tyler, RI 28775", "mail": "kellypeter@gmail.com", "id": 10, "created_at": "1974-07-10T19:02:25", "updated_at": "2005-11-07T09:09:06"}, "emitted_at": 1649892395000} diff --git a/airbyte-integrations/connectors/source-faker/source_faker/products.json b/airbyte-integrations/connectors/source-faker/source_faker/products.json new file mode 100644 index 0000000000000..fc6379603eb68 --- /dev/null +++ b/airbyte-integrations/connectors/source-faker/source_faker/products.json @@ -0,0 +1,100 @@ +[{"id":1,"make":"Mazda","model":"MX-5","year":2008,"price":2869,"created_at":"2022-02-01 17:02:19"}, +{"id":2,"make":"Mercedes-Benz","model":"C-Class","year":2009,"price":42397,"created_at":"2021-01-25 14:31:33"}, +{"id":3,"make":"Honda","model":"Accord Crosstour","year":2011,"price":63293,"created_at":"2021-02-11 05:36:03"}, +{"id":4,"make":"GMC","model":"Jimmy","year":1998,"price":34079,"created_at":"2022-01-24 03:00:03"}, +{"id":5,"make":"Infiniti","model":"FX","year":2004,"price":17036,"created_at":"2021-10-02 03:55:44"}, +{"id":6,"make":"Dodge","model":"Intrepid","year":2002,"price":65498,"created_at":"2022-01-18 00:41:08"}, +{"id":7,"make":"Nissan","model":"Frontier","year":2005,"price":14516,"created_at":"2021-04-22 16:37:44"}, +{"id":8,"make":"Chevrolet","model":"Express 1500","year":2007,"price":13023,"created_at":"2021-07-12 07:13:04"}, +{"id":9,"make":"Bentley","model":"Continental GTC","year":2008,"price":43458,"created_at":"2021-03-17 05:43:15"}, +{"id":10,"make":"Cadillac","model":"DTS","year":2008,"price":43859,"created_at":"2021-08-12 07:33:58"}, +{"id":11,"make":"Dodge","model":"Ram 2500","year":2000,"price":82904,"created_at":"2021-09-03 10:51:16"}, +{"id":12,"make":"Suzuki","model":"SJ 410","year":1984,"price":38667,"created_at":"2021-01-11 00:15:46"}, +{"id":13,"make":"Audi","model":"S4","year":2005,"price":2391,"created_at":"2021-09-06 03:31:10"}, +{"id":14,"make":"Chevrolet","model":"Suburban 2500","year":1998,"price":55733,"created_at":"2021-10-18 17:26:05"}, +{"id":15,"make":"Ford","model":"Ranger","year":2000,"price":20228,"created_at":"2022-03-24 04:03:19"}, +{"id":16,"make":"Chevrolet","model":"Corvette","year":2009,"price":75052,"created_at":"2021-12-31 03:38:21"}, +{"id":17,"make":"Mitsubishi","model":"Pajero","year":1993,"price":84058,"created_at":"2021-10-15 00:25:34"}, +{"id":18,"make":"Lincoln","model":"LS","year":2002,"price":34081,"created_at":"2022-02-14 22:12:01"}, +{"id":19,"make":"Dodge","model":"Magnum","year":2005,"price":85545,"created_at":"2021-07-25 22:49:48"}, +{"id":20,"make":"Pontiac","model":"Grand Am","year":2001,"price":54837,"created_at":"2021-10-15 14:08:30"}, +{"id":21,"make":"Chevrolet","model":"Suburban 1500","year":2006,"price":89410,"created_at":"2021-03-23 15:40:43"}, +{"id":22,"make":"GMC","model":"Sierra 1500","year":2005,"price":14288,"created_at":"2021-08-30 13:40:04"}, +{"id":23,"make":"GMC","model":"3500","year":1995,"price":12011,"created_at":"2022-04-24 13:11:08"}, +{"id":24,"make":"Mazda","model":"Mazda5","year":2006,"price":6393,"created_at":"2021-07-07 14:14:33"}, +{"id":25,"make":"Chevrolet","model":"Camaro","year":1967,"price":71590,"created_at":"2021-01-10 21:50:22"}, +{"id":26,"make":"Ford","model":"Explorer Sport Trac","year":2010,"price":23498,"created_at":"2022-04-20 00:52:20"}, +{"id":27,"make":"Dodge","model":"Caravan","year":1985,"price":50071,"created_at":"2022-01-05 10:13:31"}, +{"id":28,"make":"Nissan","model":"240SX","year":1992,"price":38379,"created_at":"2022-04-07 04:48:48"}, +{"id":29,"make":"Oldsmobile","model":"Intrigue","year":2002,"price":21376,"created_at":"2021-10-01 13:30:49"}, +{"id":30,"make":"Audi","model":"TT","year":2011,"price":40893,"created_at":"2021-02-28 23:06:37"}, +{"id":31,"make":"Ford","model":"Crown Victoria","year":2006,"price":86225,"created_at":"2021-01-28 23:33:27"}, +{"id":32,"make":"Toyota","model":"Tacoma","year":2003,"price":73558,"created_at":"2022-01-28 22:02:04"}, +{"id":33,"make":"Buick","model":"Regal","year":1994,"price":32279,"created_at":"2022-04-04 13:35:49"}, +{"id":34,"make":"Mercedes-Benz","model":"C-Class","year":2001,"price":98732,"created_at":"2021-03-30 23:16:05"}, +{"id":35,"make":"GMC","model":"Sierra 3500","year":2002,"price":48267,"created_at":"2021-07-30 20:29:51"}, +{"id":36,"make":"Pontiac","model":"G6","year":2005,"price":16766,"created_at":"2021-03-24 07:53:33"}, +{"id":37,"make":"Subaru","model":"Outback Sport","year":2002,"price":34523,"created_at":"2021-12-23 22:47:32"}, +{"id":38,"make":"Ferrari","model":"F430","year":2007,"price":31677,"created_at":"2021-01-11 04:49:57"}, +{"id":39,"make":"Mitsubishi","model":"Montero","year":2003,"price":67136,"created_at":"2021-05-10 07:37:56"}, +{"id":40,"make":"Nissan","model":"Sentra","year":1993,"price":78236,"created_at":"2021-11-10 23:48:26"}, +{"id":41,"make":"Mitsubishi","model":"3000GT","year":1993,"price":58150,"created_at":"2021-09-08 06:55:22"}, +{"id":42,"make":"Ford","model":"E350","year":2012,"price":55270,"created_at":"2021-03-24 13:17:37"}, +{"id":43,"make":"Ford","model":"Taurus","year":1987,"price":13522,"created_at":"2021-10-27 21:03:59"}, +{"id":44,"make":"Chevrolet","model":"Avalanche","year":2012,"price":9862,"created_at":"2021-07-13 12:22:26"}, +{"id":45,"make":"Dodge","model":"Charger","year":2012,"price":81887,"created_at":"2021-04-24 01:48:24"}, +{"id":46,"make":"Jaguar","model":"S-Type","year":2005,"price":34372,"created_at":"2021-04-03 08:56:17"}, +{"id":47,"make":"Plymouth","model":"Grand Voyager","year":1994,"price":90637,"created_at":"2022-04-21 09:21:08"}, +{"id":48,"make":"Pontiac","model":"6000","year":1989,"price":65165,"created_at":"2021-10-30 13:03:07"}, +{"id":49,"make":"Lexus","model":"IS","year":2006,"price":22434,"created_at":"2021-01-16 10:45:52"}, +{"id":50,"make":"Isuzu","model":"VehiCROSS","year":2001,"price":38180,"created_at":"2021-12-13 16:29:27"}, +{"id":51,"make":"Buick","model":"Regal","year":2000,"price":38680,"created_at":"2021-12-29 22:25:54"}, +{"id":52,"make":"Mercedes-Benz","model":"E-Class","year":2007,"price":51556,"created_at":"2021-07-06 11:42:23"}, +{"id":53,"make":"Buick","model":"LeSabre","year":2001,"price":10904,"created_at":"2022-01-05 18:23:35"}, +{"id":54,"make":"Porsche","model":"928","year":1989,"price":70917,"created_at":"2022-01-02 23:16:45"}, +{"id":55,"make":"Lexus","model":"RX","year":2007,"price":5212,"created_at":"2021-07-10 15:02:53"}, +{"id":56,"make":"Ford","model":"Econoline E250","year":1996,"price":75095,"created_at":"2021-02-04 16:17:18"}, +{"id":57,"make":"Chevrolet","model":"Blazer","year":2001,"price":61918,"created_at":"2021-12-08 07:25:30"}, +{"id":58,"make":"GMC","model":"Savana 3500","year":2003,"price":30307,"created_at":"2021-11-21 23:11:45"}, +{"id":59,"make":"BMW","model":"M","year":2002,"price":24598,"created_at":"2021-05-28 04:08:53"}, +{"id":60,"make":"Saturn","model":"S-Series","year":1992,"price":96288,"created_at":"2021-08-24 04:43:43"}, +{"id":61,"make":"Chrysler","model":"Sebring","year":2003,"price":34753,"created_at":"2021-02-11 11:25:35"}, +{"id":62,"make":"Lotus","model":"Evora","year":2010,"price":42760,"created_at":"2021-08-31 00:29:05"}, +{"id":63,"make":"Jeep","model":"Wrangler","year":2011,"price":8684,"created_at":"2021-06-24 10:38:05"}, +{"id":64,"make":"Ford","model":"Expedition","year":2012,"price":25653,"created_at":"2021-07-01 16:13:20"}, +{"id":65,"make":"Chevrolet","model":"Avalanche 2500","year":2006,"price":3158,"created_at":"2021-08-14 10:55:13"}, +{"id":66,"make":"Mazda","model":"Mazda3","year":2012,"price":79820,"created_at":"2021-05-25 21:55:52"}, +{"id":67,"make":"Toyota","model":"Tacoma","year":2005,"price":73572,"created_at":"2021-01-22 09:56:02"}, +{"id":68,"make":"Ford","model":"Explorer Sport","year":2000,"price":64579,"created_at":"2021-02-16 06:56:06"}, +{"id":69,"make":"GMC","model":"Savana Cargo Van","year":2006,"price":65944,"created_at":"2021-09-12 14:08:53"}, +{"id":70,"make":"Chevrolet","model":"HHR","year":2009,"price":8953,"created_at":"2021-08-17 04:25:43"}, +{"id":71,"make":"Ford","model":"Bronco II","year":1989,"price":41811,"created_at":"2021-07-14 14:20:28"}, +{"id":72,"make":"Chevrolet","model":"Suburban 2500","year":2011,"price":57488,"created_at":"2021-09-22 12:32:57"}, +{"id":73,"make":"Suzuki","model":"Grand Vitara","year":2008,"price":6408,"created_at":"2021-11-12 23:19:52"}, +{"id":74,"make":"Mazda","model":"Mazda6","year":2012,"price":14805,"created_at":"2021-06-01 01:55:32"}, +{"id":75,"make":"Chevrolet","model":"Tahoe","year":1998,"price":33585,"created_at":"2022-01-09 04:28:54"}, +{"id":76,"make":"Ford","model":"Explorer Sport Trac","year":2010,"price":2087,"created_at":"2022-03-28 00:28:16"}, +{"id":77,"make":"Ford","model":"F150","year":2007,"price":17621,"created_at":"2021-03-23 15:08:10"}, +{"id":78,"make":"Ford","model":"Taurus","year":1995,"price":16478,"created_at":"2021-06-07 22:29:50"}, +{"id":79,"make":"Mitsubishi","model":"Truck","year":1992,"price":70616,"created_at":"2022-01-30 05:14:02"}, +{"id":80,"make":"Dodge","model":"Colt","year":1994,"price":34163,"created_at":"2022-04-02 18:06:30"}, +{"id":81,"make":"Mazda","model":"RX-7","year":1991,"price":29634,"created_at":"2021-01-06 10:30:59"}, +{"id":82,"make":"Pontiac","model":"Grand Prix","year":1984,"price":88575,"created_at":"2021-02-24 06:06:57"}, +{"id":83,"make":"Mazda","model":"Mazdaspeed 3","year":2012,"price":77723,"created_at":"2021-11-11 22:48:05"}, +{"id":84,"make":"Alfa Romeo","model":"Spider","year":1992,"price":64288,"created_at":"2021-01-06 03:50:27"}, +{"id":85,"make":"Audi","model":"S8","year":2002,"price":33718,"created_at":"2021-07-21 11:14:54"}, +{"id":86,"make":"Isuzu","model":"Amigo","year":1992,"price":53335,"created_at":"2022-03-02 10:42:21"}, +{"id":87,"make":"Toyota","model":"Paseo","year":1996,"price":74558,"created_at":"2021-10-02 14:54:58"}, +{"id":88,"make":"Lincoln","model":"Continental Mark VII","year":1986,"price":42150,"created_at":"2021-10-02 04:48:53"}, +{"id":89,"make":"Dodge","model":"Dakota","year":1997,"price":64516,"created_at":"2021-09-09 23:13:26"}, +{"id":90,"make":"Chevrolet","model":"Tahoe","year":1998,"price":51461,"created_at":"2021-04-06 08:29:19"}, +{"id":91,"make":"Pontiac","model":"Vibe","year":2006,"price":12134,"created_at":"2021-01-11 22:30:14"}, +{"id":92,"make":"Volkswagen","model":"Eos","year":2011,"price":53128,"created_at":"2021-01-12 23:25:06"}, +{"id":93,"make":"Mazda","model":"Mazdaspeed6","year":2007,"price":90902,"created_at":"2021-12-29 14:29:03"}, +{"id":94,"make":"Nissan","model":"Xterra","year":2005,"price":41532,"created_at":"2021-09-07 09:00:49"}, +{"id":95,"make":"Mercury","model":"Sable","year":2005,"price":71337,"created_at":"2021-01-31 22:13:44"}, +{"id":96,"make":"BMW","model":"330","year":2006,"price":14494,"created_at":"2021-09-17 20:52:48"}, +{"id":97,"make":"Audi","model":"R8","year":2008,"price":17642,"created_at":"2021-09-21 11:56:24"}, +{"id":98,"make":"Cadillac","model":"CTS-V","year":2007,"price":19914,"created_at":"2021-09-02 15:38:46"}, +{"id":99,"make":"GMC","model":"1500 Club Coupe","year":1997,"price":82288,"created_at":"2021-04-20 18:58:15"}, +{"id":100,"make":"Buick","model":"Somerset","year":1986,"price":64148,"created_at":"2021-06-10 19:07:38"}] \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-faker/source_faker/products_catalog.json b/airbyte-integrations/connectors/source-faker/source_faker/products_catalog.json new file mode 100644 index 0000000000000..484f58f8bd137 --- /dev/null +++ b/airbyte-integrations/connectors/source-faker/source_faker/products_catalog.json @@ -0,0 +1,16 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { "type": "number" }, + "make": { "type": "string" }, + "model": { "type": "string" }, + "year": { "type": "string" }, + "price": { "type": "number" }, + "created_at": { + "type": "string", + "format": "date-time", + "airbyte_type": "timestamp_without_timezone" + } + } +} diff --git a/airbyte-integrations/connectors/source-faker/source_faker/purchases_catalog.json b/airbyte-integrations/connectors/source-faker/source_faker/purchases_catalog.json new file mode 100644 index 0000000000000..7d07183a028f1 --- /dev/null +++ b/airbyte-integrations/connectors/source-faker/source_faker/purchases_catalog.json @@ -0,0 +1,24 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { "type": "number" }, + "user_id": { "type": "number" }, + "product_id": { "type": "number" }, + "added_to_cart_at": { + "type": "string", + "format": "date-time", + "airbyte_type": "timestamp_without_timezone" + }, + "purchased_at": { + "type": "string", + "format": "date-time", + "airbyte_type": "timestamp_without_timezone" + }, + "returned_at": { + "type": "string", + "format": "date-time", + "airbyte_type": "timestamp_without_timezone" + } + } +} diff --git a/airbyte-integrations/connectors/source-faker/source_faker/source.py b/airbyte-integrations/connectors/source-faker/source_faker/source.py index b7534654f6f61..5873e31e88662 100644 --- a/airbyte-integrations/connectors/source-faker/source_faker/source.py +++ b/airbyte-integrations/connectors/source-faker/source_faker/source.py @@ -3,9 +3,10 @@ # +import datetime import json import os -from datetime import datetime +import random from typing import Dict, Generator from airbyte_cdk.logger import AirbyteLogger @@ -59,12 +60,23 @@ def discover(self, logger: AirbyteLogger, config: Dict[str, any]) -> AirbyteCata by their names and types) """ streams = [] + dirname = os.path.dirname(os.path.realpath(__file__)) # Fake Users - dirname = os.path.dirname(os.path.realpath(__file__)) - spec_path = os.path.join(dirname, "catalog.json") + spec_path = os.path.join(dirname, "users_catalog.json") catalog = read_json(spec_path) streams.append(AirbyteStream(name="Users", json_schema=catalog, supported_sync_modes=["full_refresh", "incremental"])) + + # Fake Products + spec_path = os.path.join(dirname, "products_catalog.json") + catalog = read_json(spec_path) + streams.append(AirbyteStream(name="Products", json_schema=catalog, supported_sync_modes=["full_refresh"])) + + # Fake Purchases + spec_path = os.path.join(dirname, "purchases_catalog.json") + catalog = read_json(spec_path) + streams.append(AirbyteStream(name="Purchases", json_schema=catalog, supported_sync_modes=["full_refresh", "incremental"])) + return AirbyteCatalog(streams=streams) def read( @@ -91,25 +103,69 @@ def read( """ count: int = config["count"] if "count" in config else 0 - seed: int = config["seed"] if "seed" in config else state["seed"] if "seed" in state else None + seed: int = config["seed"] if "seed" in config else None + records_per_sync: int = config["records_per_sync"] if "records_per_sync" in config else 500 + records_per_slice: int = config["records_per_slice"] if "records_per_slice" in config else 100 + Faker.seed(seed) fake = Faker() + to_generate_users = False + to_generate_purchases = False + purchases_stream = None + purchases_count = state["Purchases"]["purchases_count"] if "Purchases" in state else 0 + for stream in catalog.streams: + if stream.stream.name == "Users": + to_generate_users = True + for stream in catalog.streams: + if stream.stream.name == "Purchases": + purchases_stream = stream + to_generate_purchases = True + + if to_generate_purchases and not to_generate_users: + raise ValueError("Purchases stream cannot be enabled without Users stream") + for stream in catalog.streams: if stream.stream.name == "Users": cursor = get_stream_cursor(state, stream.stream.name) total_records = cursor + records_in_sync = 0 + records_in_page = 0 for i in range(cursor, count): - yield AirbyteMessage( - type=Type.RECORD, - record=AirbyteRecordMessage( - stream=stream.stream.name, data=emit_user(fake, i), emitted_at=int(datetime.now().timestamp()) * 1000 - ), - ) - total_records = total_records + 1 - - yield emit_state(stream.stream.name, total_records, seed) + user = generate_user(fake, i) + yield generate_record(stream, user) + total_records += 1 + records_in_sync += 1 + records_in_page += 1 + + if to_generate_purchases: + purchases = generate_purchases(fake, user, purchases_count) + for p in purchases: + yield generate_record(purchases_stream, p) + purchases_count += 1 + + if records_in_page == records_per_slice: + yield generate_state(state, stream, {"cursor": total_records, "seed": seed}) + records_in_page = 0 + + if records_in_sync == records_per_sync: + break + + yield generate_state(state, stream, {"cursor": total_records, "seed": seed}) + if purchases_stream is not None: + yield generate_state(state, purchases_stream, {"purchases_count": purchases_count}) + + elif stream.stream.name == "Products": + products = generate_products() + for p in products: + yield generate_record(stream, p) + yield generate_state(state, stream, {"product_count": len(products)}) + + elif stream.stream.name == "Purchases": + # Purchases are generated as part of Users stream + True + else: raise ValueError(stream.stream.name) @@ -119,19 +175,28 @@ def get_stream_cursor(state: Dict[str, any], stream: str) -> int: return cursor -def emit_state(stream: str, value: int, seed: int): - message = AirbyteMessage(type=Type.STATE, state=AirbyteStateMessage(data={stream: {"cursor": value, "seed": seed}})) - return message +def generate_record(stream: any, data: any): + return AirbyteMessage( + type=Type.RECORD, + record=AirbyteRecordMessage(stream=stream.stream.name, data=data, emitted_at=int(datetime.datetime.now().timestamp()) * 1000), + ) + +def generate_state(state: Dict[str, any], stream: any, data: any): + state[ + stream.stream.name + ] = data # since we have multiple streams, we need to build up the "combined state" for all streams and emit that each time until the platform has support for per-stream state + return AirbyteMessage(type=Type.STATE, state=AirbyteStateMessage(data=state)) -def emit_user(fake: Faker, idx: int): + +def generate_user(fake: Faker, user_id: int): profile = fake.profile() del profile["birthdate"] # the birthdate field seems to not obey the seed at the moment, so we'll ignore it time_a = fake.date_time() time_b = fake.date_time() metadata = { - "id": idx, + "id": user_id + 1, "created_at": time_a if time_a <= time_b else time_b, "updated_at": time_a if time_a > time_b else time_b, } @@ -139,6 +204,50 @@ def emit_user(fake: Faker, idx: int): return profile +def generate_purchases(fake: Faker, user: any, purchases_count: int) -> list[Dict]: + purchases: list[Dict] = [] + purchase_percent_remaining = 80 # ~ 20% of people will have no purchases + total_products = len(generate_products()) + purchase_percent_remaining = purchase_percent_remaining - random.randrange(1, 100) + i = 0 + while purchase_percent_remaining > 0: + id = purchases_count + i + 1 + product_id = random.randrange(1, total_products) + added_to_cart_at = random_date_in_range(user["created_at"]) + purchased_at = ( + random_date_in_range(added_to_cart_at) if added_to_cart_at is not None and random.randrange(1, 100) <= 70 else None + ) # 70% likely to purchase the item in the cart + returned_at = ( + random_date_in_range(purchased_at) if purchased_at is not None and random.randrange(1, 100) <= 15 else None + ) # 15% likely to return the item + purchase = { + "id": id, + "product_id": product_id, + "user_id": user["id"], + "added_to_cart_at": added_to_cart_at, + "purchased_at": purchased_at, + "returned_at": returned_at, + } + purchases.append(purchase) + + purchase_percent_remaining = purchase_percent_remaining - random.randrange(1, 100) + i += 1 + return purchases + + +def generate_products() -> list[Dict]: + dirname = os.path.dirname(os.path.realpath(__file__)) + return read_json(os.path.join(dirname, "products.json")) + + def read_json(filepath): with open(filepath, "r") as f: return json.loads(f.read()) + + +def random_date_in_range(start_date: datetime.datetime, end_date: datetime.datetime = datetime.datetime.now()) -> datetime.datetime: + time_between_dates = end_date - start_date + days_between_dates = time_between_dates.days + random_number_of_days = random.randrange(days_between_dates) + random_date = start_date + datetime.timedelta(days=random_number_of_days) + return random_date diff --git a/airbyte-integrations/connectors/source-faker/source_faker/spec.json b/airbyte-integrations/connectors/source-faker/source_faker/spec.json index f4f58007bcea8..c056a36a07121 100644 --- a/airbyte-integrations/connectors/source-faker/source_faker/spec.json +++ b/airbyte-integrations/connectors/source-faker/source_faker/spec.json @@ -9,10 +9,10 @@ "properties": { "count": { "title": "Count", - "description": "How many fake records should be generated", + "description": "How many users should be generated in total. This setting does not apply to the purchases or products stream.", "type": "integer", "minimum": 1, - "default": 100, + "default": 1000, "order": 0 }, "seed": { @@ -21,6 +21,22 @@ "type": "integer", "default": -1, "order": 1 + }, + "records_per_sync": { + "title": "Records Per Sync", + "description": "How many fake records will be returned for each sync, for each stream? By default, it will take 2 syncs to create the requested 1000 records.", + "type": "integer", + "minimum": 1, + "default": 500, + "order": 2 + }, + "records_per_slice": { + "title": "Records Per Stream Slice", + "description": "How many fake records will be in each page (stream slice), before a state message is emitted?", + "type": "integer", + "minimum": 1, + "default": 100, + "order": 3 } } } diff --git a/airbyte-integrations/connectors/source-faker/source_faker/catalog.json b/airbyte-integrations/connectors/source-faker/source_faker/users_catalog.json similarity index 100% rename from airbyte-integrations/connectors/source-faker/source_faker/catalog.json rename to airbyte-integrations/connectors/source-faker/source_faker/users_catalog.json diff --git a/airbyte-integrations/connectors/source-faker/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-faker/unit_tests/unit_test.py index d2c3db0dd103e..93df146d0c0db 100644 --- a/airbyte-integrations/connectors/source-faker/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-faker/unit_tests/unit_test.py @@ -3,6 +3,7 @@ # import jsonschema +import pytest from airbyte_cdk.models import AirbyteMessage, ConfiguredAirbyteCatalog, Type from source_faker import SourceFaker @@ -14,7 +15,7 @@ def test_source_streams(): catalog = AirbyteMessage(type=Type.CATALOG, catalog=catalog).dict(exclude_unset=True) schemas = [stream["json_schema"] for stream in catalog["catalog"]["streams"]] - assert len(schemas) == 1 + assert len(schemas) == 3 assert schemas[0]["properties"] == { "id": {"type": "number"}, "created_at": {"type": "string", "format": "date-time", "airbyte_type": "timestamp_without_timezone"}, @@ -37,7 +38,7 @@ def test_source_streams(): jsonschema.Draft7Validator.check_schema(schema) -def test_read_random_data(): +def test_read_small_random_data(): source = SourceFaker() logger = None config = {"count": 10} @@ -49,14 +50,101 @@ def test_read_random_data(): record_rows_count = 0 state_rows_count = 0 + latest_state = {} for row in iterator: if row.type is Type.RECORD: record_rows_count = record_rows_count + 1 if row.type is Type.STATE: state_rows_count = state_rows_count + 1 + latest_state = row assert record_rows_count == 10 assert state_rows_count == 1 + assert latest_state.state.data == {"Users": {"cursor": 10, "seed": None}} + + +def test_read_big_random_data(): + source = SourceFaker() + logger = None + config = {"count": 1000, "records_per_slice": 100, "records_per_sync": 1000} + catalog = ConfiguredAirbyteCatalog( + streams=[ + {"stream": {"name": "Users", "json_schema": {}}, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite"}, + {"stream": {"name": "Products", "json_schema": {}}, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite"}, + ] + ) + state = {} + iterator = source.read(logger, config, catalog, state) + + record_rows_count = 0 + state_rows_count = 0 + latest_state = {} + for row in iterator: + if row.type is Type.RECORD: + record_rows_count = record_rows_count + 1 + if row.type is Type.STATE: + state_rows_count = state_rows_count + 1 + latest_state = row + + assert record_rows_count == 1000 + 100 # 1000 users, and 100 products + assert state_rows_count == 10 + 1 + 1 # 1000/100 + one more state at the end, and one state for the products + assert latest_state.state.data == {"Products": {"product_count": 100}, "Users": {"cursor": 1000, "seed": None}} + + +def test_with_purchases(): + source = SourceFaker() + logger = None + config = {"count": 1000, "records_per_sync": 1000} + catalog = ConfiguredAirbyteCatalog( + streams=[ + {"stream": {"name": "Users", "json_schema": {}}, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite"}, + {"stream": {"name": "Products", "json_schema": {}}, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite"}, + {"stream": {"name": "Purchases", "json_schema": {}}, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite"}, + ] + ) + state = {} + iterator = source.read(logger, config, catalog, state) + + record_rows_count = 0 + state_rows_count = 0 + latest_state = {} + for row in iterator: + if row.type is Type.RECORD: + record_rows_count = record_rows_count + 1 + if row.type is Type.STATE: + state_rows_count = state_rows_count + 1 + latest_state = row + + assert record_rows_count > 1000 + 100 # should be greater than 1000 users, and 100 products + assert state_rows_count > 10 + 1 + 1 # should be greater than 1000/100 + one more state at the end, and one state for the products + assert latest_state.state.data["Users"] == {"cursor": 1000, "seed": None} + assert latest_state.state.data["Products"] == {"product_count": 100} + assert latest_state.state.data["Purchases"]["purchases_count"] > 0 + + +def test_sync_ends_with_limit(): + source = SourceFaker() + logger = None + config = {"count": 100, "records_per_sync": 5} + catalog = ConfiguredAirbyteCatalog( + streams=[{"stream": {"name": "Users", "json_schema": {}}, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite"}] + ) + state = {} + iterator = source.read(logger, config, catalog, state) + + record_rows_count = 0 + state_rows_count = 0 + latest_state = {} + for row in iterator: + if row.type is Type.RECORD: + record_rows_count = record_rows_count + 1 + if row.type is Type.STATE: + state_rows_count = state_rows_count + 1 + latest_state = row + + assert record_rows_count == 5 + assert state_rows_count == 1 + assert latest_state.state.data == {"Users": {"cursor": 5, "seed": None}} def test_read_with_seed(): @@ -76,3 +164,18 @@ def test_read_with_seed(): records = [row for row in iterator if row.type is Type.RECORD] assert records[0].record.data["company"] == "Gibson-Townsend" assert records[0].record.data["mail"] == "zamoradenise@yahoo.com" + + +def test_ensure_no_purchases_without_users(): + with pytest.raises(ValueError): + source = SourceFaker() + logger = None + config = {"count": 100} + catalog = ConfiguredAirbyteCatalog( + streams=[ + {"stream": {"name": "Purchases", "json_schema": {}}, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite"}, + ] + ) + state = {} + iterator = source.read(logger, config, catalog, state) + iterator.__next__() diff --git a/airbyte-webapp-e2e-tests/cypress.json b/airbyte-webapp-e2e-tests/cypress.json index 02259a2ab4e4f..a1021faa7c7fa 100644 --- a/airbyte-webapp-e2e-tests/cypress.json +++ b/airbyte-webapp-e2e-tests/cypress.json @@ -8,7 +8,7 @@ "destination.spec.js", "source.spec.js" ], - "viewportHeight": 800, + "viewportHeight": 800, "viewportWidth": 1280, "retries": { "runMode": 2, diff --git a/docs/integrations/sources/faker.md b/docs/integrations/sources/faker.md index ffffcb01af14d..a9cce5ec5bd46 100644 --- a/docs/integrations/sources/faker.md +++ b/docs/integrations/sources/faker.md @@ -41,5 +41,7 @@ N/A | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- | :-------------------------------------------------------- | +| 0.1.3 | 2022-05-26 | [13248](https://github.com/airbytehq/airbyte/pull/13248) | Add options for records_per_sync and page_size | +| 0.1.2 | 2022-05-26 | [13248](https://github.com/airbytehq/airbyte/pull/13293) | Test publication flow | | 0.1.1 | 2022-05-26 | [13235](https://github.com/airbytehq/airbyte/pull/13235) | Publish for AMD and ARM (M1 Macs) & remove User.birthdate | | 0.1.0 | 2022-04-12 | [11738](https://github.com/airbytehq/airbyte/pull/11738) | The Faker Source is created |