#750

/capstor/store/cscs/swissai/infra01/vision-datasets/raw/sft/nemotron_image_training_v3/datasets

kind	webdataset (zst)
status	active
samples	n/a
size	526.2 GB
files	11
first seen	2026-07-22 14:04
last seen	2026-07-22 14:04
registered	2026-07-22 14:04

# dataset #750
path: /capstor/store/cscs/swissai/infra01/vision-datasets/raw/sft/nemotron_image_training_v3/datasets
kind: webdataset (zst) | status: active
samples: n/a | size: 526.2 GB | files: 11

fields:
- key: int|text
- out: text
- sh: text
- jpg: image

tree (truncated):
```
aokvqa.tar.zst
cc3m.tar.zst
ccpdf.tar.zst
figureqa.tar.zst
fintabnet.tar.zst
flickr30k.tar.zst
geometry3k.tar.zst
geomverse.tar.zst
gqa_1.tar.zst
hiertext.tar.zst
plotqa_2.tar.zst
```

samples


  wrote train-00000.parquet (19200 rows, 3.34 GB)
DONE: 19200 rows packed, 0 rows skipped (missing images)
#!/bin/bash
#SBATCH --job-name=pack_aokvqa_1
#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --cpus-per-task=8
#SBATCH --time=01:30:00
#SBATCH --output=/capstor/store/cscs/swissai/infra01/vision-datasets/raw/sft/nemotron_image_training_v3/datasets/aokvqa_1/pack_%j.out
#SBATCH --error=/capstor/store/cscs/swissai/infra01/vision-datasets/raw/sft/nemotron_image_training_v3/datasets/aokvqa_1/pack_%j.err
#SBATCH --account=infra01
#SBATCH --partition=debug
#SBATCH --environment=nemo

set -euo pipefail
pip install --quiet pyarrow

BASE=/capstor/store/cscs/swissai/infra01/vision-datasets/raw/sft/nemotron_image_training_v3
HF=$BASE/hf___nvidia___Nemotron-Image-Training-v3
SWISS=$BASE/swissai___Nemotron-Image-Training-v3

python $BASE/datasets/aokvqa_1/mirror.py \
  --jsonl     $HF/aokvqa_2/aokvqa_2.jsonl \
  --media-dir $BASE/datasets/aokvqa_1/train2017 \
  --out-dir   $SWISS/aokvqa_2
000000273893
000000527994
000000308310
000000127496
000000320785
000000483833
000000116358
000000062276
000000479102
000000236386
000000003580
000000334474
000000535699
000000536149
000000025283
000000509484
000000381908
000000523306
000000279078
000000541687
000000504541
000000010720
000000479880
000000336980
000000192927
000000422956
000000073860
000000138405
000000468793
000000007653

#	key	out	sh
1	aokvqa_1_2051322	wrote train-00000.parquet (19200 rows, 3.34 GB) DONE: 19200 rows packed, 0 rows skipped (missing images)
2	mirror		#!/bin/bash #SBATCH --job-name=pack_aokvqa_1 #SBATCH --nodes=1 #SBATCH --ntasks-per-node=1 #SBATCH --cpus-per-task=8 #SBATCH --time=01:30:00 #SBATCH --output=/capstor/store/cscs/swissai/infra01/vision-datasets/raw/sft/nemotron_image_training_v3/datasets/aokvqa_1/pack_%j.out #SBATCH --error=/capstor/store/cscs/swissai/infra01/vision-datasets/raw/sft/nemotron_image_training_v3/datasets/aokvqa_1/pack_%j.err #SBATCH --account=infra01 #SBATCH --partition=debug #SBATCH --environment=nemo set -euo pipefail pip install --quiet pyarrow BASE=/capstor/store/cscs/swissai/infra01/vision-datasets/raw/sft/nemotron_image_training_v3 HF=$BASE/hf___nvidia___Nemotron-Image-Training-v3 SWISS=$BASE/swissai___Nemotron-Image-Training-v3 python $BASE/datasets/aokvqa_1/mirror.py \ --jsonl $HF/aokvqa_2/aokvqa_2.jsonl \ --media-dir $BASE/datasets/aokvqa_1/train2017 \ --out-dir $SWISS/aokvqa_2
3	000000273893
4	000000527994
5	000000308310
6	000000127496
7	000000320785
8	000000483833
9	000000116358
10	000000062276
11	000000479102
12	000000236386
13	000000003580
14	000000334474
15	000000535699
16	000000536149
17	000000025283
18	000000509484
19	000000381908
20	000000523306
21	000000279078
22	000000541687
23	000000504541
24	000000010720
25	000000479880
26	000000336980
27	000000192927
28	000000422956
29	000000073860
30	000000138405
31	000000468793
32	000000007653