Update scripts

tools/executorch

@@ -3,6 +3,8 @@
 set -e
 pushd $(cd $(dirname ${0})/..; pwd) > /dev/null
 
+src_dir=./third_party/source/executorch
+
 case "${OSTYPE}" in
     msys*) python="winpty python";;
     *) python="python";;
@@ -10,22 +12,27 @@ esac
 
 clean() {
     echo "executorch clean"
-    rm -rf "./third_party/source/executorch"
+    rm -rf "${src_dir}"
 }
 
 sync() {
     echo "executorch sync"
-    [ -d "./third_party/source/executorch" ] || git clone --quiet https://github.com/pytorch/executorch.git "./third_party/source/executorch"
-    git -C "./third_party/source/executorch" pull --quiet --prune
-    # mkdir -p "./third_party/source/executorch/schema"
-    # curl --silent --show-error --location --output "./third_party/source/executorch/schema/scalar_type.fbs" "https://github.com/pytorch/executorch/raw/main/schema/scalar_type.fbs"
-    # curl --silent --show-error --location --output "./third_party/source/executorch/schema/program.fbs" "https://github.com/pytorch/executorch/raw/main/schema/program.fbs"
+    if [ ! -d "${src_dir}" ]; then
+        git clone --quiet --depth=1 --branch main --single-branch https://github.com/pytorch/executorch.git "${src_dir}"
+    else
+        git -C "${src_dir}" fetch --quiet --depth=1 origin main
+        git -C "${src_dir}" reset --quiet --hard FETCH_HEAD
+        git -C "${src_dir}" gc --quiet --prune=all
+    fi
+    # mkdir -p "${src_dir}/schema"
+    # curl --silent --show-error --location --output "${src_dir}/schema/scalar_type.fbs" "https://github.com/pytorch/executorch/raw/main/schema/scalar_type.fbs"
+    # curl --silent --show-error --location --output "${src_dir}/schema/program.fbs" "https://github.com/pytorch/executorch/raw/main/schema/program.fbs"
 }
 
 schema() {
     echo "executorch schema"
     [[ $(grep -U $'\x0D' ./source/executorch-schema.js) ]] && crlf=1
-    node ./tools/flatc.js --out ./source/executorch-schema.js ./third_party/source/executorch/schema/program.fbs ./third_party/source/executorch/backends/xnnpack/serialization/schema.fbs ./third_party/source/executorch/backends/vulkan/serialization/schema.fbs
+    node ./tools/flatc.js --out ./source/executorch-schema.js ${src_dir}/schema/program.fbs ${src_dir}/backends/xnnpack/serialization/schema.fbs ${src_dir}/backends/vulkan/serialization/schema.fbs
     if [[ -n ${crlf} ]]; then
         unix2dos --quiet --newfile ./source/executorch-schema.js ./source/executorch-schema.js
     fi

tools/gguf

@@ -3,20 +3,21 @@
 set -e
 pushd $(cd $(dirname ${0})/..; pwd) > /dev/null
 
+src_dir="./third_party/source/llama.cpp"
+
 clean() {
     echo "ggml clean"
-    rm -rf "./third_party/source/llama.cpp"
+    rm -rf "${src_dir}"
 }
 
 sync() {
     echo "ggml sync"
-    dir="./third_party/source/llama.cpp"
-    if [ ! -d "${dir}" ]; then
-        git clone --quiet https://github.com/ggerganov/llama.cpp.git "${dir}"
+    if [ ! -d "${src_dir}" ]; then
+        git clone --quiet --depth=1 --branch master --single-branch https://github.com/ggerganov/llama.cpp.git "${src_dir}"
     else
-        pushd "${dir}" > /dev/null
-        git pull --quiet --prune
-        popd > /dev/null
+        git -C "${src_dir}" fetch --quiet --depth=1 origin master
+        git -C "${src_dir}" reset --quiet --hard FETCH_HEAD
+        git -C "${src_dir}" gc --quiet --prune=all
     fi
 }
 

tools/pytorch

@@ -3,13 +3,15 @@
 set -e
 pushd $(cd $(dirname ${0})/..; pwd) > /dev/null
 
+env_dir=./third_party/env/pytorch
+src_dir=./third_party/source/pytorch
+
 case "${OSTYPE}" in
     msys*) python="winpty python";;
     *) python="python";;
 esac
 
 venv() {
-    env_dir=./third_party/env/pytorch
     [ -d "${env_dir}" ] || ${python} -m venv ${env_dir}
     case "${OSTYPE}" in
         msys*) source ${env_dir}/Scripts/activate;;
@@ -20,14 +22,19 @@ venv() {
 
 clean() {
     echo "pytorch clean"
+    rm -rf "${src_dir}"
     rm -rf "./third_party/source/caffe2"
-    rm -rf "./third_party/source/pytorch"
 }
 
 sync() {
     echo "pytorch sync"
-    [ -d "./third_party/source/pytorch" ] || git clone --quiet https://github.com/pytorch/pytorch.git "./third_party/source/pytorch"
-    git -C "./third_party/source/pytorch" pull --quiet --prune
+    if [ ! -d "${src_dir}" ]; then
+        git clone --quiet --depth=1 --branch main --single-branch https://github.com/pytorch/pytorch.git "${src_dir}"
+    else
+        git -C "${src_dir}" fetch --quiet --depth=1 origin main
+        git -C "${src_dir}" reset --quiet --hard FETCH_HEAD
+        git -C "${src_dir}" gc --quiet --prune=all
+    fi
     mkdir -p "./third_party/source/caffe2/proto"
     curl --silent --show-error --location --output "./third_party/source/caffe2/proto/caffe2.proto" "https://github.com/pytorch/pytorch/raw/5e69e11d098a2cfccc8a59377c431e9c71cab9a8/caffe2/proto/caffe2.proto"
     curl --silent --show-error --location --output "./third_party/source/caffe2/proto/torch.proto" "https://github.com/pytorch/pytorch/raw/5e69e11d098a2cfccc8a59377c431e9c71cab9a8/caffe2/proto/torch.proto"

tools/sklearn

@@ -3,13 +3,16 @@
 set -e
 pushd $(cd $(dirname ${0})/..; pwd) > /dev/null
 
+env_dir=./third_party/env/scikit-learn
+src_dir=./third_party/source/scikit-learn
+
 case "${OSTYPE}" in
     msys*) python="winpty python";;
     *) python="python";;
 esac
 
 venv() {
-    env_dir=./third_party/env/scikit-learn
+    env_dir=${env_dir}
     [ -d "${env_dir}" ] || ${python} -m venv ${env_dir}
     case "${OSTYPE}" in
         msys*) source ${env_dir}/Scripts/activate;;
@@ -20,18 +23,19 @@ venv() {
 
 clean() {
     echo "sklearn clean"
-    rm -rf "./third_party/env/scikit-learn"
-    rm -rf "./third_party/source/scikit-learn"
+    rm -rf "${env_dir}"
+    rm -rf "${src_dir}"
 }
 
 sync() {
     echo "sklearn sync"
-    [ -d "./third_party/source/scikit-learn" ] || git clone --quiet --recursive "https://github.com/scikit-learn/scikit-learn.git" "./third_party/source/scikit-learn"
-    pushd "./third_party/source/scikit-learn" > /dev/null
-    git pull --quiet --prune
-    git submodule sync --quiet
-    git submodule update --quiet --init --recursive
-    popd > /dev/null
+    if [ ! -d "${src_dir}" ]; then
+        git clone --quiet --depth=1 --branch main --single-branch https://github.com/scikit-learn/scikit-learn.git "${src_dir}"
+    else
+        git -C "${src_dir}" fetch --quiet --depth=1 origin main
+        git -C "${src_dir}" reset --quiet --hard FETCH_HEAD
+        git -C "${src_dir}" gc --quiet --prune=all
+    fi
 }
 
 install() {

tools/tf

@@ -3,13 +3,15 @@
 set -e
 pushd $(cd $(dirname ${0})/..; pwd) > /dev/null
 
+env_dir=./third_party/env/tensorflow
+src_dir=./third_party/source/tensorflow
+
 case "${OSTYPE}" in
     msys*) python="winpty python";;
     *) python="python";;
 esac
 
 venv() {
-    env_dir=./third_party/env/tensorflow
     [ -d "${env_dir}" ] || ${python} -m venv ${env_dir}
     case "${OSTYPE}" in
         msys*) source ${env_dir}/Scripts/activate;;
@@ -20,17 +22,20 @@ venv() {
 
 clean() {
     echo "tf clean"
-    rm -rf "./third_party/env/tensorflow"
-    rm -rf "./third_party/source/tensorflow"
+    rm -rf "${env_dir}"
+    rm -rf "${src_dir}"
     rm -rf "./third_party/source/tflite-support"
 }
 
 sync() {
     echo "tf sync"
-    if [ ! -d "./third_party/source/tensorflow" ]; then
-        git clone --quiet https://github.com/tensorflow/tensorflow.git "./third_party/source/tensorflow"
+    if [ ! -d "${src_dir}" ]; then
+        git clone --quiet --depth=1 --branch master --single-branch https://github.com/tensorflow/tensorflow.git "${src_dir}"
+    else
+        git -C "${src_dir}" fetch --quiet --depth=1 origin master
+        git -C "${src_dir}" reset --quiet --hard FETCH_HEAD
+        git -C "${src_dir}" gc --quiet --prune=all
     fi
-    git -C "./third_party/source/tensorflow" pull --quiet --prune
     mkdir -p "./third_party/source/tflite-support/tensorflow_lite_support/metadata"
     curl --silent --show-error --location --output "./third_party/source/tflite-support/tensorflow_lite_support/metadata/metadata_schema.fbs" "https://github.com/tensorflow/tflite-support/raw/master/tensorflow_lite_support/metadata/metadata_schema.fbs"
 }
@@ -46,7 +51,7 @@ install() {
 schema() {
     echo "tf schema"
     [[ $(grep -U $'\x0D' ./source/tf-proto.js) ]] && crlf=1
-    node ./tools/protoc.js --binary --text --json --root tf --out ./source/tf-proto.js --path ./third_party/source/tensorflow --path ./third_party/source/tensorflow/third_party/xla tensorflow/core/protobuf/saved_model.proto tensorflow/core/protobuf/tensor_bundle.proto tensorflow/core/util/saved_tensor_slice.proto tensorflow/core/util/event.proto tensorflow/core/protobuf/config.proto tensorflow/core/util/memmapped_file_system.proto tensorflow/core/protobuf/fingerprint.proto
+    node ./tools/protoc.js --binary --text --json --root tf --out ./source/tf-proto.js --path ${src_dir} --path ${src_dir}/third_party/xla tensorflow/core/protobuf/saved_model.proto tensorflow/core/protobuf/tensor_bundle.proto tensorflow/core/util/saved_tensor_slice.proto tensorflow/core/util/event.proto tensorflow/core/protobuf/config.proto tensorflow/core/util/memmapped_file_system.proto tensorflow/core/protobuf/fingerprint.proto
     if [[ -n ${crlf} ]]; then
         unix2dos --quiet --newfile ./source/tf-proto.js ./source/tf-proto.js
     fi
@@ -54,13 +59,13 @@ schema() {
     [[ $(grep -U $'\x0D' ./source/tflite-schema.js) ]] && crlf=1
     temp1=$(mktemp -d)
     temp2=$(mktemp -d)
-    node ./tools/flatc.js --text --root tflite --out ./source/tflite-schema.js ./third_party/source/tensorflow/tensorflow/compiler/mlir/lite/schema/schema.fbs ./third_party/source/tflite-support/tensorflow_lite_support/metadata/metadata_schema.fbs
+    node ./tools/flatc.js --text --root tflite --out ./source/tflite-schema.js ${src_dir}/tensorflow/compiler/mlir/lite/schema/schema.fbs ./third_party/source/tflite-support/tensorflow_lite_support/metadata/metadata_schema.fbs
     if [[ -n ${crlf} ]]; then
         unix2dos --quiet --newfile ./source/tflite-schema.js ./source/tflite-schema.js
     fi
     echo "keras schema"
     [[ $(grep -U $'\x0D' ./source/keras-proto.js) ]] && crlf=1
-    node ./tools/protoc.js --binary --text --root tf --out ./source/keras-proto.js --path ./third_party/source/tensorflow --path ./third_party/source/tensorflow/third_party/xla/third_party/tsl tensorflow/python/keras/protobuf/saved_metadata.proto
+    node ./tools/protoc.js --binary --text --root tf --out ./source/keras-proto.js --path ${src_dir} --path ${src_dir}/third_party/xla/third_party/tsl tensorflow/python/keras/protobuf/saved_metadata.proto
     if [[ -n ${crlf} ]]; then
         unix2dos --quiet --newfile ./source/keras-proto.js ./source/keras-proto.js
     fi