Skip to content

Chore: Moved test datasets to s3 #2109

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Mar 10, 2025
Merged

Chore: Moved test datasets to s3 #2109

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
246a026
Moved test datasets to s3
kevinmessiaen Feb 12, 2025
0fbf312
Fixed notebook
kevinmessiaen Mar 6, 2025
9795f3b
Merge branch 'main' into feature/move-test-data-s3
kevinmessiaen Mar 6, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 10 additions & 21 deletions docs/reference/notebooks/amazon_review_classification_sklearn.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -117,15 +117,10 @@
]
},
{
"metadata": {},
"cell_type": "code",
"execution_count": 3,
"metadata": {
"ExecuteTime": {
"end_time": "2023-11-08T20:55:58.032511Z",
"start_time": "2023-11-08T20:55:57.792680Z"
}
},
"outputs": [],
"execution_count": null,
"source": [
"# Constants.\n",
"RANDOM_SEED = 0\n",
Expand All @@ -135,8 +130,8 @@
"TARGET_NAME = \"isHelpful\"\n",
"\n",
"# Paths.\n",
"DATA_URL = \"ftp://sys.giskard.ai/pub/unit_test_resources/amazon_review_dataset/reviews.json\"\n",
"DATA_PATH = Path.home() / \".giskard\" / \"amazon_review_dataset\" / \"reviews.json\""
"DATA_URL = \"https://giskard-library-test-datasets.s3.eu-north-1.amazonaws.com/amazon_review_dataset-reviews.json.tar.gz\"\n",
"DATA_PATH = Path.home() / \".giskard\" / \"amazon_review_dataset\" / \"reviews.json.tar.gz\""
]
},
{
Expand All @@ -156,18 +151,12 @@
]
},
{
"metadata": {},
"cell_type": "code",
"execution_count": 4,
"metadata": {
"ExecuteTime": {
"end_time": "2023-11-08T20:56:00.293536Z",
"start_time": "2023-11-08T20:56:00.234306Z"
},
"collapsed": false
},
"outputs": [],
"execution_count": null,
"source": [
"def fetch_from_ftp(url: str, file: Path) -> None:\n",
"def fetch_demo_data(url: str, file: Path) -> None:\n",
" \"\"\"Helper to fetch data from the FTP server.\"\"\"\n",
" if not file.parent.exists():\n",
" file.parent.mkdir(parents=True, exist_ok=True)\n",
Expand All @@ -181,7 +170,7 @@
"\n",
"def download_data(**kwargs) -> pd.DataFrame:\n",
" \"\"\"Download the dataset using URL.\"\"\"\n",
" fetch_from_ftp(DATA_URL, DATA_PATH)\n",
" fetch_demo_data(DATA_URL, DATA_PATH)\n",
" _df = pd.read_json(DATA_PATH, lines=True, **kwargs)\n",
" return _df\n",
"\n",
Expand Down Expand Up @@ -215,10 +204,10 @@
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"cell_type": "code",
"outputs": [],
"execution_count": null,
"source": [
"reviews_df = download_data()\n",
"reviews_df = preprocess_data(reviews_df)"
Expand Down
22 changes: 8 additions & 14 deletions docs/reference/notebooks/drug_classification_sklearn.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,17 +97,10 @@
]
},
{
"metadata": {},
"cell_type": "code",
"execution_count": 2,
"id": "d44430add2918aa1",
"metadata": {
"ExecuteTime": {
"end_time": "2024-02-09T09:29:15.513819Z",
"start_time": "2024-02-09T09:29:15.470284Z"
},
"collapsed": false
},
"outputs": [],
"execution_count": null,
"source": [
"# Constants.\n",
"RANDOM_SEED = 0\n",
Expand All @@ -121,9 +114,10 @@
"NA_TO_K_CATEGORIES = ['<10', '10-20', '20-30', '>30']\n",
"\n",
"# Paths.\n",
"DATA_URL = \"ftp://sys.giskard.ai/pub/unit_test_resources/drug_classification_dataset/drug200.csv\"\n",
"DATA_PATH = Path.home() / \".giskard\" / \"drug_classification_dataset\" / \"drug200.csv\""
]
"DATA_URL = \"https://giskard-library-test-datasets.s3.eu-north-1.amazonaws.com/drug_classification_dataset-drug200.csv.tar.gz\"\n",
"DATA_PATH = Path.home() / \".giskard\" / \"drug_classification_dataset\" / \"drug200.csv.tar.gz\""
],
"id": "a161e40415287e1f"
},
{
"cell_type": "markdown",
Expand Down Expand Up @@ -158,7 +152,7 @@
},
"outputs": [],
"source": [
"def fetch_from_ftp(url: str, file: Path) -> None:\n",
"def fetch_demo_data(url: str, file: Path) -> None:\n",
" \"\"\"Helper to fetch data from the FTP server.\"\"\"\n",
" if not file.parent.exists():\n",
" file.parent.mkdir(parents=True, exist_ok=True)\n",
Expand All @@ -172,7 +166,7 @@
"\n",
"def load_data() -> pd.DataFrame:\n",
" \"\"\"Load data.\"\"\"\n",
" fetch_from_ftp(DATA_URL, DATA_PATH)\n",
" fetch_demo_data(DATA_URL, DATA_PATH)\n",
" df = pd.read_csv(DATA_PATH)\n",
" return df\n",
"\n",
Expand Down
17 changes: 9 additions & 8 deletions docs/reference/notebooks/fake_real_news_classification.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,9 @@
"source": [
"import os\n",
"import string\n",
"import tarfile\n",
"from pathlib import Path\n",
"from typing import Tuple, Callable\n",
"from urllib.request import urlretrieve\n",
"\n",
"import numpy as np\n",
Expand All @@ -79,7 +81,6 @@
"from nltk.corpus import stopwords\n",
"from sklearn.metrics import accuracy_score\n",
"from sklearn.model_selection import train_test_split\n",
"from typing import Tuple, Callable\n",
"\n",
"from giskard import Dataset, Model, scan, testing"
]
Expand Down Expand Up @@ -142,7 +143,7 @@
"RANDOM_SEED = 0\n",
"\n",
"# Paths.\n",
"DATA_URL = \"ftp://sys.giskard.ai/pub/unit_test_resources/fake_real_news_dataset/{}\"\n",
"DATA_URL = \"https://giskard-library-test-datasets.s3.eu-north-1.amazonaws.com/fake_real_news_dataset-{}\"\n",
"DATA_PATH = Path.home() / \".giskard\" / \"fake_real_news_dataset\""
]
},
Expand Down Expand Up @@ -170,7 +171,7 @@
},
"outputs": [],
"source": [
"def fetch_from_ftp(url: str, file: Path) -> None:\n",
"def fetch_demo_data(url: str, file: Path) -> None:\n",
" \"\"\"Helper to fetch data from the FTP server.\"\"\"\n",
" if not file.parent.exists():\n",
" file.parent.mkdir(parents=True, exist_ok=True)\n",
Expand All @@ -184,15 +185,15 @@
"\n",
"def fetch_dataset() -> None:\n",
" \"\"\"Gradually fetch all necessary files from the FTP server.\"\"\"\n",
" files_to_fetch = (\"Fake.csv\", \"True.csv\", \"glove_100d.txt\")\n",
" files_to_fetch = (\"Fake.csv.tar.gz\", \"True.csv.tar.gz\", \"glove_100d.txt.tar.gz\")\n",
" for file_name in files_to_fetch:\n",
" fetch_from_ftp(DATA_URL.format(file_name), DATA_PATH / file_name)\n",
" fetch_demo_data(DATA_URL.format(file_name), DATA_PATH / file_name)\n",
"\n",
"\n",
"def load_data(**kwargs) -> pd.DataFrame:\n",
" \"\"\"Load data.\"\"\"\n",
" real_df = pd.read_csv(DATA_PATH / \"True.csv\", **kwargs)\n",
" fake_df = pd.read_csv(DATA_PATH / \"Fake.csv\", **kwargs)\n",
" real_df = pd.read_csv(DATA_PATH / \"True.csv.tar.gz\", **kwargs)\n",
" fake_df = pd.read_csv(DATA_PATH / \"Fake.csv.tar.gz\", **kwargs)\n",
"\n",
" # Create target column.\n",
" real_df[TARGET_COLUMN_NAME] = 0\n",
Expand Down Expand Up @@ -380,7 +381,7 @@
"def get_embeddings_matrix() -> np.ndarray:\n",
" \"\"\"Create matrix, where each row is an embedding of a specific word.\"\"\"\n",
" # Load glove embeddings.\n",
" embeddings_dict = dict(parse_line(*line.rstrip().rsplit(' ')) for line in open(DATA_PATH / \"glove_100d.txt\"))\n",
" embeddings_dict = dict(parse_line(*line.rstrip().rsplit(' ')) for line in tarfile.open(DATA_PATH / \"glove_100d.txt.tar.gz\", \"r:gz\").extractfile(\"fake_real_news_dataset-glove_100d.txt\").read().decode())\n",
"\n",
" # Create embeddings matrix with glove word vectors.\n",
" embeddings_matrix = init_embeddings_matrix(embeddings_dict)\n",
Expand Down
18 changes: 6 additions & 12 deletions docs/reference/notebooks/hotel_text_regression.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,24 +91,18 @@
]
},
{
"metadata": {},
"cell_type": "code",
"execution_count": 2,
"metadata": {
"ExecuteTime": {
"end_time": "2023-11-09T12:12:05.303464Z",
"start_time": "2023-11-09T12:12:05.254149Z"
},
"collapsed": false
},
"outputs": [],
"execution_count": null,
"source": [
"# Constants.\n",
"FEATURE_COLUMN_NAME = \"Full_Review\"\n",
"TARGET_COLUMN_NAME = \"Reviewer_Score\"\n",
"\n",
"# Paths.\n",
"DATA_URL = \"ftp://sys.giskard.ai/pub/unit_test_resources/hotel_text_regression_dataset/Hotel_Reviews.csv\"\n",
"DATA_PATH = Path.home() / \".giskard\" / \"hotel_text_regression_dataset\" / \"Hotel_Reviews.csv\""
"DATA_URL = \"https://giskard-library-test-datasets.s3.eu-north-1.amazonaws.com/hotel_text_regression_dataset-Hotel_Reviews.csv.tar.gz\"\n",
"DATA_PATH = Path.home() / \".giskard\" / \"hotel_text_regression_dataset\" / \"Hotel_Reviews.csv.tar.gz\""
]
},
{
Expand Down Expand Up @@ -142,7 +136,7 @@
},
"outputs": [],
"source": [
"def fetch_from_ftp(url: str, file: Path) -> None:\n",
"def fetch_demo_data(url: str, file: Path) -> None:\n",
" \"\"\"Helper to fetch data from the FTP server.\"\"\"\n",
" if not file.parent.exists():\n",
" file.parent.mkdir(parents=True, exist_ok=True)\n",
Expand All @@ -155,7 +149,7 @@
"\n",
"\n",
"def load_data(**kwargs) -> pd.DataFrame:\n",
" fetch_from_ftp(DATA_URL, DATA_PATH)\n",
" fetch_demo_data(DATA_URL, DATA_PATH)\n",
" df = pd.read_csv(DATA_PATH, **kwargs)\n",
"\n",
" # Create target column.\n",
Expand Down
42 changes: 12 additions & 30 deletions docs/reference/notebooks/ieee_fraud_detection_adversarial_validation.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,23 +102,17 @@
]
},
{
"metadata": {},
"cell_type": "code",
"execution_count": 2,
"metadata": {
"ExecuteTime": {
"end_time": "2023-11-09T12:17:44.751420Z",
"start_time": "2023-11-09T12:17:44.719440Z"
},
"collapsed": false
},
"outputs": [],
"execution_count": null,
"source": [
"# Constants.\n",
"TARGET_COLUMN = 'isTest'\n",
"IDX_LABEL = 'TransactionID'\n",
"\n",
"# Paths.\n",
"DATA_URL = \"ftp://sys.giskard.ai/pub/unit_test_resources/fraud_detection_classification_dataset/{}\"\n",
"DATA_URL = \"https://giskard-library-test-datasets.s3.eu-north-1.amazonaws.com/fraud_detection_classification_dataset-{}\"\n",
"DATA_PATH = Path.home() / \".giskard\" / \"fraud_detection_classification_dataset\""
]
},
Expand All @@ -141,18 +135,12 @@
]
},
{
"metadata": {},
"cell_type": "code",
"execution_count": 3,
"metadata": {
"ExecuteTime": {
"end_time": "2023-11-09T12:17:45.925766Z",
"start_time": "2023-11-09T12:17:45.904823Z"
},
"collapsed": false
},
"outputs": [],
"execution_count": null,
"source": [
"def fetch_from_ftp(url: str, file: Path) -> None:\n",
"def fetch_demo_data(url: str, file: Path) -> None:\n",
" \"\"\"Helper to fetch data from the FTP server.\"\"\"\n",
" if not file.parent.exists():\n",
" file.parent.mkdir(parents=True, exist_ok=True)\n",
Expand All @@ -165,9 +153,9 @@
"\n",
"\n",
"def fetch_dataset():\n",
" files_to_fetch = [\"train_transaction.csv\", \"train_identity.csv\", \"test_transaction.csv\", \"test_identity.csv\"]\n",
" files_to_fetch = [\"train_transaction.csv.tar.gz\", \"train_identity.csv.tar.gz\", \"test_transaction.csv.tar.gz\", \"test_identity.csv.tar.gz\"]\n",
" for file_name in files_to_fetch:\n",
" fetch_from_ftp(DATA_URL.format(file_name), DATA_PATH / file_name)\n",
" fetch_demo_data(DATA_URL.format(file_name), DATA_PATH / file_name)\n",
"\n",
"\n",
"# Define data-types of transactions features.\n",
Expand Down Expand Up @@ -225,11 +213,11 @@
"def read_set(_type):\n",
" \"\"\"Read both transactions and identity data.\"\"\"\n",
" print(f\"Reading transactions data...\")\n",
" _df = pd.read_csv(os.path.join(DATA_PATH, f'{_type}_transaction.csv'),\n",
" _df = pd.read_csv(os.path.join(DATA_PATH, f'{_type}_transaction.csv.tar.gz'),\n",
" index_col=IDX_LABEL, dtype=DATA_TYPES_TRANSACTION, nrows=250)\n",
"\n",
" print(f\"Reading identity data...\")\n",
" _df = _df.join(pd.read_csv(os.path.join(DATA_PATH, f'{_type}_identity.csv'),\n",
" _df = _df.join(pd.read_csv(os.path.join(DATA_PATH, f'{_type}_identity.csv.tar.gz'),\n",
" index_col=IDX_LABEL, dtype=DATA_TYPES_ID))\n",
" return _df\n",
"\n",
Expand All @@ -248,16 +236,10 @@
]
},
{
"metadata": {},
"cell_type": "code",
"execution_count": 4,
"metadata": {
"ExecuteTime": {
"end_time": "2023-11-09T12:17:46.316557Z",
"start_time": "2023-11-09T12:17:46.290804Z"
},
"collapsed": false
},
"outputs": [],
"execution_count": null,
"source": [
"def preprocess_dataset(train_set, test_set):\n",
" \"\"\"Unite train and test into common dataframe.\"\"\"\n",
Expand Down
8 changes: 4 additions & 4 deletions docs/reference/notebooks/insurance_prediction_lgbm.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -179,8 +179,8 @@
"CATEGORICAL_COLS = [\"sex\", \"smoker\", \"region\"]\n",
"\n",
"# Paths.\n",
"DATA_URL = \"ftp://sys.giskard.ai/pub/unit_test_resources/insurance_prediction_dataset/us_health_insurance_dataset.csv\"\n",
"DATA_PATH = Path.home() / \".giskard\" / \"insurance_prediction_dataset\" / \"us_health_insurance_dataset.csv\""
"DATA_URL = \"https://giskard-library-test-datasets.s3.eu-north-1.amazonaws.com/insurance_prediction_dataset-us_health_insurance_dataset.csv.tar.gz\"\n",
"DATA_PATH = Path.home() / \".giskard\" / \"insurance_prediction_dataset\" / \"us_health_insurance_dataset.csv.tar.gz\""
]
},
{
Expand Down Expand Up @@ -216,7 +216,7 @@
},
"outputs": [],
"source": [
"def fetch_from_ftp(url: str, file: Path) -> None:\n",
"def fetch_demo_data(url: str, file: Path) -> None:\n",
" \"\"\"Helper to fetch data from the FTP server.\"\"\"\n",
" if not file.parent.exists():\n",
" file.parent.mkdir(parents=True, exist_ok=True)\n",
Expand All @@ -230,7 +230,7 @@
"\n",
"def download_data(**kwargs) -> pd.DataFrame:\n",
" \"\"\"Download the dataset using URL.\"\"\"\n",
" fetch_from_ftp(DATA_URL, DATA_PATH)\n",
" fetch_demo_data(DATA_URL, DATA_PATH)\n",
" _df = pd.read_csv(DATA_PATH, **kwargs)\n",
" return _df"
]
Expand Down
Loading
Loading