#!/bin/bash
#SBATCH --exclusive
#SBATCH --mem=0
#SBATCH --overcommit

# Copyright (c) 2019 NVIDIA CORPORATION. All rights reserved.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -eux

readonly docker_image="nvcr.io/nvidia/tensorflow:19.08-py3"
readonly datadir="/raid/data/bert"
readonly checkpointdir="$PWD/checkpoints"

readonly mounts=".:/workspace/bert,${datadir}:/workspace/bert/data,${checkpointdir}:/results"

DO_LOWER_CASE=${DO_LOWER_CASE:-1}
if [ "$DO_LOWER_CASE" == "1" ]; then
  CASING_DIR_PREFIX="uncased"
else
  CASING_DIR_PREFIX="cased"
fi

DO_BERT_BASE=${DO_BERT_BASE:-1}
if [ "$DO_BERT_BASE" == "1" ]; then
  CASING_DIR_SUFFIX="L-12_H-768_A-12"
else
  CASING_DIR_SUFFIX="L-24_H-1024_A-16"
fi

srun --ntasks="${SLURM_JOB_NUM_NODES}" --ntasks-per-node=1 mkdir -p "${checkpointdir}/biobert_phase_1"
srun --ntasks="${SLURM_JOB_NUM_NODES}" --ntasks-per-node=1 mkdir -p "${checkpointdir}/biobert_phase_2"

PHASE1="\
     --train_batch_size=${BATCHSIZE:-128} \
     --learning_rate=${LEARNING_RATE:-3.2e-5} \
     --num_accumulation_steps=${NUM_ACCUMULATION_STEPS:-128} \
     --input_files_dir=lower_case_${DO_LOWER_CASE}_seq_len_128_max_pred_20_masked_lm_prob_0.15_random_seed_12345_dupe_factor_5_shard_1472_test_split_10/pubmed_baseline/training \
     --eval_files_dir=lower_case_${DO_LOWER_CASE}_seq_len_128_max_pred_20_masked_lm_prob_0.15_random_seed_12345_dupe_factor_5_shard_1472_test_split_10/pubmed_baseline/test \
     --max_seq_length=128 \
     --max_predictions_per_seq=20 \
     --num_train_steps=19531 \
     --num_warmup_steps=1953 \
     --output_dir=/results/biobert_phase_1 \
     "

PHASE2="\
     --train_batch_size=${BATCHSIZE:-16} \
     --learning_rate=${LEARNING_RATE:-6.4e-5} \
     --num_accumulation_steps=${NUM_ACCUMULATION_STEPS:-512} \
     --input_files_dir=/workspace/bert/data/tfrecord/lower_case_${DO_LOWER_CASE}_seq_len_512_max_pred_80_masked_lm_prob_0.15_random_seed_12345_dupe_factor_5_shard_1472_test_split_10/pubmed_baseline/training \
     --eval_files_dir=/workspace/bert/data/tfrecord/lower_case_${DO_LOWER_CASE}_seq_len_512_max_pred_80_masked_lm_prob_0.15_random_seed_12345_dupe_factor_5_shard_1472_test_split_10/pubmed_baseline/test \
     --max_seq_length=512 \
     --max_predictions_per_seq=80 \
     --num_train_steps=4340 \
     --num_warmup_steps=434 \
     --output_dir=/results/biobert_phase_2 \
     --init_checkpoint=/results/biobert_phase_1/model.ckpt-19531 \
    "

PHASES=( "$PHASE1" "$PHASE2" )

PHASE=${PHASE:-1}

BERT_CMD="\
    python /workspace/bert/run_pretraining.py \
     ${PHASES[$((PHASE-1))]} \
     --bert_config_file=/workspace/bert/data/download/google_pretrained_weights/${CASING_DIR_PREFIX}_${CASING_DIR_SUFFIX}/bert_config.json \
     --vocab_file=/workspace/bert/data/download/google_pretrained_weights/${CASING_DIR_PREFIX}_${CASING_DIR_SUFFIX}/vocab.txt \
     --do_train=True \
     --do_eval=True \
     --save_checkpoints_steps=5000 \
     --horovod --amp --use_xla \
     --allreduce_post_accumulation=True \
     --eval_batch_size=8"

srun --mpi=pmi2 -l --container-image="${docker_image}" --container-mounts="${mounts}" bash -c "${BERT_CMD}"