Upload run_cloud_300m.sh with huggingface_hub

run_cloud_300m.sh

@@ -0,0 +1,137 @@
+#!/usr/bin/env bash
+# =============================================================================
+#  LUNA 300M — Cloud Smoke Test + Full Training (Single Command)
+#
+#  Paste this ONE command into any cloud GPU terminal:
+#
+#    export HF_TOKEN=hf_xxx && \
+#    git clone https://huggingface.co/spaces/ASTERIZER/LUNA /workspace/LUNA && \
+#    cd /workspace/LUNA && bash run_cloud_300m.sh
+#
+#  What it does:
+#    1. Installs all dependencies
+#    2. Downloads the 4.5B token dataset from HuggingFace
+#    3. Runs a 2-step smoke test to verify everything works
+#    4. If smoke test passes, starts the full 300M pretraining
+#    5. Uploads all checkpoints to HuggingFace when done
+#
+#  The smoke test catches ALL issues (model init, data loading, forward/backward,
+#  tokenizer, checkpoint saving, text generation) BEFORE burning GPU hours.
+# =============================================================================
+
+set -e
+
+DATA_DIR="/workspace/data/litdata_pretrain_final"
+OUT_DIR="/workspace/out/pretrain/luna-300m"
+HF_DATASET="ASTERIZER/Luna_Dataset"
+MAX_TOKENS="${1:-4515286950}"
+
+echo ""
+echo "============================================================"
+echo "  LUNA 300M — Cloud Pipeline"
+echo "  Dataset : $HF_DATASET -> $DATA_DIR"
+echo "  Output  : $OUT_DIR"
+echo "  Tokens  : $MAX_TOKENS"
+echo "============================================================"
+
+# ── 1. Dependencies ──────────────────────────────────────────────────────────
+echo ""
+echo "[1/5] Installing dependencies..."
+pip install -q --upgrade pip 2>/dev/null
+pip install -q \
+    torch torchvision \
+    psutil \
+    pyyaml \
+    huggingface_hub \
+    transformers \
+    datasets \
+    pyarrow \
+    gdown \
+    tensorboard 2>/dev/null || true
+echo "      Done."
+
+# ── 2. Download Dataset ──────────────────────────────────────────────────────
+echo ""
+echo "[2/5] Fetching dataset from HuggingFace..."
+if [ -f "$DATA_DIR/index.json" ]; then
+    echo "      Dataset already exists at $DATA_DIR, skipping download."
+else
+    HF_TOKEN="${HF_TOKEN:-}"
+    python fetch_data.py \
+        --source huggingface \
+        --hf_repo "$HF_DATASET" \
+        --out_dir "$DATA_DIR" \
+        --hf_token "$HF_TOKEN"
+fi
+
+# Quick verify
+DATA_DIR_ENV="$DATA_DIR" python - <<'PYEOF'
+import json, os
+from pathlib import Path
+data_dir = Path(os.environ["DATA_DIR_ENV"])
+idx_path = data_dir / "index.json"
+if not idx_path.exists():
+    print(f"FATAL: No index.json at {data_dir}")
+    raise SystemExit(1)
+with open(idx_path) as f:
+    idx = json.load(f)
+chunks = idx.get("chunks", [])
+total = sum(c.get("dim", 0) for c in chunks)
+present = sum(1 for c in chunks if (data_dir / c["filename"]).exists())
+missing = len(chunks) - present
+print(f"      Dataset: {len(chunks)} chunks, {total:,} tokens, {missing} missing")
+if missing > 0:
+    print("FATAL: Missing chunks!")
+    raise SystemExit(1)
+PYEOF
+echo "      Dataset verified."
+
+# ── 3. Smoke Test ─────────────────────────────────────────────────────────────
+echo ""
+echo "[3/5] Running 2-step smoke test..."
+echo "      (This verifies model + data + training + generation BEFORE full run)"
+echo ""
+
+python smoke_test_300m.py \
+    --data_path "$DATA_DIR" \
+    --steps 2
+
+SMOKE_EXIT=$?
+if [ $SMOKE_EXIT -ne 0 ]; then
+    echo ""
+    echo "============================================================"
+    echo "  SMOKE TEST FAILED — aborting full training."
+    echo "  Fix the issues above and retry."
+    echo "============================================================"
+    exit 1
+fi
+
+echo ""
+echo "      Smoke test passed! Starting full training..."
+
+# ── 4. Full Training ─────────────────────────────────────────────────────────
+echo ""
+echo "[4/5] Starting LUNA 300M full pretraining..."
+echo "      - Validation: every 100 steps (10 generation prompts)"
+echo "      - Checkpoints: every 1000 steps"
+echo "      - Auto VRAM probe for max batch size"
+echo ""
+
+python train_300m.py \
+    --config train_config_300m.yaml \
+    --data_path "$DATA_DIR" \
+    --out_dir "$OUT_DIR" \
+    --max_tokens "$MAX_TOKENS"
+
+# ── 5. Done ──────────────────────────────────────────────────────────────────
+echo ""
+echo "============================================================"
+echo "  LUNA 300M — COMPLETE"
+echo "  Output      : $OUT_DIR"
+echo "  Final model : $OUT_DIR/final/lit_model.pth"
+if [ -n "$HF_TOKEN" ]; then
+echo "  HF Upload   : https://huggingface.co/ASTERIZER/LUNA-300M"
+else
+echo "  HF Upload   : SKIPPED (set HF_TOKEN)"
+fi
+echo "============================================================"