chang-m-yun commited on Jun 5

Commit

0588586

verified ·

1 Parent(s): 5ea3af5

Add files using upload-large-folder tool

Browse files

This view is limited to 50 files because it contains too many changes. See raw diff

Files changed (50) hide show

.gitattributes +8 -0
fold_0/logs.models.fold_0.ENCSR543YPH/logfile.modelling.fold_0.ENCSR543YPH.batch_loss.tsv +0 -0
fold_0/logs.models.fold_0.ENCSR543YPH/logfile.modelling.fold_0.ENCSR543YPH.bias_formatting.stderr.txt +38 -0
fold_0/logs.models.fold_0.ENCSR543YPH/logfile.modelling.fold_0.ENCSR543YPH.chrombpnet_formatting.stderr.txt +40 -0
fold_0/logs.models.fold_0.ENCSR543YPH/logfile.modelling.fold_0.ENCSR543YPH.chrombpnet_formatting.stdout.txt +1 -0
fold_0/logs.models.fold_0.ENCSR543YPH/logfile.modelling.fold_0.ENCSR543YPH.chrombpnet_no_bias_formatting.stdout.txt +1 -0
fold_0/logs.models.fold_0.ENCSR543YPH/logfile.modelling.fold_0.ENCSR543YPH.stdout.txt +3 -0
fold_0/logs.models.fold_0.ENCSR543YPH/logfile.modelling.fold_0.ENCSR543YPH.stdout_v1.txt +3 -0
fold_0/model.bias_scaled.fold_0.ENCSR543YPH.h5 +3 -0
fold_0/model.bias_scaled.fold_0.ENCSR543YPH.tar +3 -0
fold_0/model.chrombpnet.fold_0.ENCSR543YPH.h5 +3 -0
fold_0/model.chrombpnet.fold_0.ENCSR543YPH.tar +3 -0
fold_0/model.chrombpnet_nobias.fold_0.ENCSR543YPH.h5 +3 -0
fold_0/model.chrombpnet_nobias.fold_0.ENCSR543YPH.tar +3 -0
fold_1/logs.models.fold_1.ENCSR543YPH/logfile.modelling.fold_1.ENCSR543YPH.stdout.txt +3 -0
fold_1/logs.models.fold_1.ENCSR543YPH/logfile.modelling.fold_1.ENCSR543YPH.stdout_v1.txt +3 -0
fold_1/model.bias_scaled.fold_1.ENCSR543YPH.h5 +3 -0
fold_1/model.bias_scaled.fold_1.ENCSR543YPH.tar +3 -0
fold_1/model.chrombpnet.fold_1.ENCSR543YPH.h5 +3 -0
fold_1/model.chrombpnet.fold_1.ENCSR543YPH.tar +3 -0
fold_1/model.chrombpnet_nobias.fold_1.ENCSR543YPH.h5 +3 -0
fold_1/model.chrombpnet_nobias.fold_1.ENCSR543YPH.tar +3 -0
fold_2/logs.models.fold_2.ENCSR543YPH/logfile.modelling.fold_2.ENCSR543YPH.args.json +23 -0
fold_2/logs.models.fold_2.ENCSR543YPH/logfile.modelling.fold_2.ENCSR543YPH.batch_loss.tsv +0 -0
fold_2/logs.models.fold_2.ENCSR543YPH/logfile.modelling.fold_2.ENCSR543YPH.bias_formatting.stderr.txt +38 -0
fold_2/logs.models.fold_2.ENCSR543YPH/logfile.modelling.fold_2.ENCSR543YPH.bias_formatting.stdout.txt +1 -0
fold_2/logs.models.fold_2.ENCSR543YPH/logfile.modelling.fold_2.ENCSR543YPH.chrombpnet.params.json +11 -0
fold_2/logs.models.fold_2.ENCSR543YPH/logfile.modelling.fold_2.ENCSR543YPH.chrombpnet_data_params.tsv +3 -0
fold_2/logs.models.fold_2.ENCSR543YPH/logfile.modelling.fold_2.ENCSR543YPH.chrombpnet_formatting.stderr.txt +40 -0
fold_2/logs.models.fold_2.ENCSR543YPH/logfile.modelling.fold_2.ENCSR543YPH.chrombpnet_formatting.stdout.txt +1 -0
fold_2/logs.models.fold_2.ENCSR543YPH/logfile.modelling.fold_2.ENCSR543YPH.chrombpnet_model_params.tsv +9 -0
fold_2/logs.models.fold_2.ENCSR543YPH/logfile.modelling.fold_2.ENCSR543YPH.chrombpnet_no_bias_formatting.stderr.txt +1 -0
fold_2/logs.models.fold_2.ENCSR543YPH/logfile.modelling.fold_2.ENCSR543YPH.chrombpnet_no_bias_formatting.stdout.txt +1 -0
fold_2/logs.models.fold_2.ENCSR543YPH/logfile.modelling.fold_2.ENCSR543YPH.epoch_loss.csv +15 -0
fold_2/logs.models.fold_2.ENCSR543YPH/logfile.modelling.fold_2.ENCSR543YPH.stderr.txt +336 -0
fold_2/logs.models.fold_2.ENCSR543YPH/logfile.modelling.fold_2.ENCSR543YPH.stdout.txt +3 -0
fold_2/logs.models.fold_2.ENCSR543YPH/logfile.modelling.fold_2.ENCSR543YPH.stdout_v1.txt +3 -0
fold_2/model.bias_scaled.fold_2.ENCSR543YPH.h5 +3 -0
fold_2/model.bias_scaled.fold_2.ENCSR543YPH.tar +3 -0
fold_2/model.chrombpnet.fold_2.ENCSR543YPH.h5 +3 -0
fold_2/model.chrombpnet.fold_2.ENCSR543YPH.tar +3 -0
fold_2/model.chrombpnet_nobias.fold_2.ENCSR543YPH.h5 +3 -0
fold_2/model.chrombpnet_nobias.fold_2.ENCSR543YPH.tar +3 -0
fold_3/logs.models.fold_3.ENCSR543YPH/logfile.modelling.fold_3.ENCSR543YPH.stdout.txt +3 -0
fold_3/logs.models.fold_3.ENCSR543YPH/logfile.modelling.fold_3.ENCSR543YPH.stdout_v1.txt +3 -0
fold_3/model.bias_scaled.fold_3.ENCSR543YPH.h5 +3 -0
fold_3/model.bias_scaled.fold_3.ENCSR543YPH.tar +3 -0
fold_3/model.chrombpnet.fold_3.ENCSR543YPH.h5 +3 -0
fold_3/model.chrombpnet.fold_3.ENCSR543YPH.tar +3 -0
fold_3/model.chrombpnet_nobias.fold_3.ENCSR543YPH.h5 +3 -0

.gitattributes CHANGED Viewed

@@ -33,3 +33,11 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text

 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+fold_2/logs.models.fold_2.ENCSR543YPH/logfile.modelling.fold_2.ENCSR543YPH.stdout.txt filter=lfs diff=lfs merge=lfs -text
+fold_0/logs.models.fold_0.ENCSR543YPH/logfile.modelling.fold_0.ENCSR543YPH.stdout.txt filter=lfs diff=lfs merge=lfs -text
+fold_2/logs.models.fold_2.ENCSR543YPH/logfile.modelling.fold_2.ENCSR543YPH.stdout_v1.txt filter=lfs diff=lfs merge=lfs -text
+fold_0/logs.models.fold_0.ENCSR543YPH/logfile.modelling.fold_0.ENCSR543YPH.stdout_v1.txt filter=lfs diff=lfs merge=lfs -text
+fold_1/logs.models.fold_1.ENCSR543YPH/logfile.modelling.fold_1.ENCSR543YPH.stdout_v1.txt filter=lfs diff=lfs merge=lfs -text
+fold_1/logs.models.fold_1.ENCSR543YPH/logfile.modelling.fold_1.ENCSR543YPH.stdout.txt filter=lfs diff=lfs merge=lfs -text
+fold_3/logs.models.fold_3.ENCSR543YPH/logfile.modelling.fold_3.ENCSR543YPH.stdout.txt filter=lfs diff=lfs merge=lfs -text
+fold_3/logs.models.fold_3.ENCSR543YPH/logfile.modelling.fold_3.ENCSR543YPH.stdout_v1.txt filter=lfs diff=lfs merge=lfs -text

fold_0/logs.models.fold_0.ENCSR543YPH/logfile.modelling.fold_0.ENCSR543YPH.batch_loss.tsv ADDED Viewed

The diff for this file is too large to render. See raw diff

fold_0/logs.models.fold_0.ENCSR543YPH/logfile.modelling.fold_0.ENCSR543YPH.bias_formatting.stderr.txt ADDED Viewed

	@@ -0,0 +1,38 @@

+INFO:    underlay of /etc/localtime required more than 50 (88) bind mounts
+INFO:    underlay of /usr/bin/nvidia-smi required more than 50 (355) bind mounts
+2023-07-16 23:07:36.378132: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-07-16 23:07:39.925981: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set
+2023-07-16 23:07:39.930874: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1
+2023-07-16 23:07:39.963122: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties:
+pciBusID: 0000:83:00.0 name: Tesla P100-PCIE-16GB computeCapability: 6.0
+coreClock: 1.3285GHz coreCount: 56 deviceMemorySize: 15.89GiB deviceMemoryBandwidth: 681.88GiB/s
+2023-07-16 23:07:39.963222: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-07-16 23:07:39.993043: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2023-07-16 23:07:39.993216: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2023-07-16 23:07:40.008804: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
+2023-07-16 23:07:40.015861: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
+2023-07-16 23:07:40.042667: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
+2023-07-16 23:07:40.049587: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
+2023-07-16 23:07:40.051113: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+2023-07-16 23:07:40.053163: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
+2023-07-16 23:07:40.053586: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+2023-07-16 23:07:40.054683: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set
+2023-07-16 23:07:40.054982: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties:
+pciBusID: 0000:83:00.0 name: Tesla P100-PCIE-16GB computeCapability: 6.0
+coreClock: 1.3285GHz coreCount: 56 deviceMemorySize: 15.89GiB deviceMemoryBandwidth: 681.88GiB/s
+2023-07-16 23:07:40.055027: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-07-16 23:07:40.055066: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2023-07-16 23:07:40.055092: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2023-07-16 23:07:40.055117: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
+2023-07-16 23:07:40.055142: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
+2023-07-16 23:07:40.055167: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
+2023-07-16 23:07:40.055191: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
+2023-07-16 23:07:40.055216: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+2023-07-16 23:07:40.055616: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
+2023-07-16 23:07:40.057307: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-07-16 23:07:42.298711: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix:
+2023-07-16 23:07:42.298813: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267]      0
+2023-07-16 23:07:42.298834: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0:   N
+2023-07-16 23:07:42.302415: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 14957 MB memory) -> physical GPU (device: 0, name: Tesla P100-PCIE-16GB, pci bus id: 0000:83:00.0, compute capability: 6.0)
+2023-07-16 23:07:43.311011: W tensorflow/python/util/util.cc:348] Sets are not currently considered sequences, but this may change in the future, so consider avoiding using them.

fold_0/logs.models.fold_0.ENCSR543YPH/logfile.modelling.fold_0.ENCSR543YPH.chrombpnet_formatting.stderr.txt ADDED Viewed

	@@ -0,0 +1,40 @@

+INFO:    underlay of /etc/localtime required more than 50 (88) bind mounts
+INFO:    underlay of /usr/bin/nvidia-smi required more than 50 (355) bind mounts
+2023-07-15 02:10:52.200538: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-07-15 02:10:55.235548: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set
+2023-07-15 02:10:55.240846: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1
+2023-07-15 02:10:55.288013: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties:
+pciBusID: 0000:82:00.0 name: Tesla P100-PCIE-16GB computeCapability: 6.0
+coreClock: 1.3285GHz coreCount: 56 deviceMemorySize: 15.89GiB deviceMemoryBandwidth: 681.88GiB/s
+2023-07-15 02:10:55.288140: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-07-15 02:10:55.317681: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2023-07-15 02:10:55.317853: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2023-07-15 02:10:55.333454: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
+2023-07-15 02:10:55.339975: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
+2023-07-15 02:10:55.365826: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
+2023-07-15 02:10:55.371790: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
+2023-07-15 02:10:55.373112: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+2023-07-15 02:10:55.395237: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
+2023-07-15 02:10:55.395668: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+2023-07-15 02:10:55.396600: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set
+2023-07-15 02:10:55.412345: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties:
+pciBusID: 0000:82:00.0 name: Tesla P100-PCIE-16GB computeCapability: 6.0
+coreClock: 1.3285GHz coreCount: 56 deviceMemorySize: 15.89GiB deviceMemoryBandwidth: 681.88GiB/s
+2023-07-15 02:10:55.412485: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-07-15 02:10:55.412565: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2023-07-15 02:10:55.412595: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2023-07-15 02:10:55.412624: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
+2023-07-15 02:10:55.412652: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
+2023-07-15 02:10:55.412680: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
+2023-07-15 02:10:55.412708: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
+2023-07-15 02:10:55.412737: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+2023-07-15 02:10:55.414779: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
+2023-07-15 02:10:55.416136: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-07-15 02:10:57.249162: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix:
+2023-07-15 02:10:57.249286: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267]      0
+2023-07-15 02:10:57.249318: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0:   N
+2023-07-15 02:10:57.252525: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 14957 MB memory) -> physical GPU (device: 0, name: Tesla P100-PCIE-16GB, pci bus id: 0000:82:00.0, compute capability: 6.0)
+2023-07-15 02:10:59.707432: W tensorflow/python/util/util.cc:348] Sets are not currently considered sequences, but this may change in the future, so consider avoiding using them.
+/opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/layers/core.py:1059: UserWarning:  is not loaded, but a Lambda layer uses it. It may cause errors.
+  , UserWarning)

fold_0/logs.models.fold_0.ENCSR543YPH/logfile.modelling.fold_0.ENCSR543YPH.chrombpnet_formatting.stdout.txt ADDED Viewed

	@@ -0,0 +1 @@


1	+ singularity exec --nv /home/groups/akundaje/anusri/simg/tf-atlas_gcp-modeling.sif python get_new_tf_model_format.py -i /oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR543YPH//chrombpnet_model_encsr880cub_bias/chrombpnet.h5 -o /oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR543YPH//chrombpnet_model_encsr880cub_bias/new_model_formats/chrombpnet

fold_0/logs.models.fold_0.ENCSR543YPH/logfile.modelling.fold_0.ENCSR543YPH.chrombpnet_no_bias_formatting.stdout.txt ADDED Viewed

	@@ -0,0 +1 @@

+ singularity exec --nv /home/groups/akundaje/anusri/simg/tf-atlas_gcp-modeling.sif python get_new_tf_model_format.py -i /oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR543YPH//chrombpnet_model_encsr880cub_bias/chrombpnet_wo_bias.h5 -o /oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR543YPH//chrombpnet_model_encsr880cub_bias/new_model_formats/chrombpnet_wo_bias

fold_0/logs.models.fold_0.ENCSR543YPH/logfile.modelling.fold_0.ENCSR543YPH.stdout.txt ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:a3c883a7833502647d7702600355d8ed6efa0590f1e2299456e724ee6fa2e018
+size 13402271

fold_0/logs.models.fold_0.ENCSR543YPH/logfile.modelling.fold_0.ENCSR543YPH.stdout_v1.txt ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:954ac81c020c6556f1d5532734b3276b9e635c7e983c766477271f06aeddad05
+size 13400665

fold_0/model.bias_scaled.fold_0.ENCSR543YPH.h5 ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:51e36a89f03bce854ad71feb31d49f1250a13eee464588fc744d1a2a701efbf4
+size 2688440

fold_0/model.bias_scaled.fold_0.ENCSR543YPH.tar ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:ce38f41ff510e8c30c7ddd132dc5a2ad6f673f09da6bcd5570ce81f14bd50303
+size 1198080

fold_0/model.chrombpnet.fold_0.ENCSR543YPH.h5 ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:df8fa8089ba20602186f01c997347721031e5e6ceeb2fa25e1c5ac96c716a15b
+size 26447928

fold_0/model.chrombpnet.fold_0.ENCSR543YPH.tar ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:51fe88f54573d0b4a411558074c837c6f61456aed7e20ddd654936a2736072c2
+size 27607040

fold_0/model.chrombpnet_nobias.fold_0.ENCSR543YPH.h5 ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:1b11650ed0755cf455f26ffae371a1bec6f52b001aa98fc78c76b08ed14d2c62
+size 25583536

fold_0/model.chrombpnet_nobias.fold_0.ENCSR543YPH.tar ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:2a7310feb3b7912162777912bafff975484915b151b21b365edc2d9bec5646ca
+size 26081280

fold_1/logs.models.fold_1.ENCSR543YPH/logfile.modelling.fold_1.ENCSR543YPH.stdout.txt ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:563f0ae5f3fd3c3cc0c1d61f5fc70eb8cd504368bc9683f96c081bc1abf98314
+size 10934105

fold_1/logs.models.fold_1.ENCSR543YPH/logfile.modelling.fold_1.ENCSR543YPH.stdout_v1.txt ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:762617e0cfdb04701fe66b8670bc175aba414d52636787b804e5653333185e61
+size 10932432

fold_1/model.bias_scaled.fold_1.ENCSR543YPH.h5 ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:391c5dc4ba5310deca3e953946cc9d0bd5bbee5fa40b7ba0d963859aa3e302b1
+size 2688440

fold_1/model.bias_scaled.fold_1.ENCSR543YPH.tar ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:8498cb16cf479e2f5395447b6efa86085d8b02b904a04febc094eb9e0c0176a7
+size 1198080

fold_1/model.chrombpnet.fold_1.ENCSR543YPH.h5 ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:38b1a11d3fab9ffecf9219855c2f2c3944f70016d0440338b99e3f63ef92ac27
+size 26447928

fold_1/model.chrombpnet.fold_1.ENCSR543YPH.tar ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:44637e12dcd93242511f654488c058c2b6d1b1132792a023a162c8376ffd6621
+size 27607040

fold_1/model.chrombpnet_nobias.fold_1.ENCSR543YPH.h5 ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:d1885c2c9d81c2a7313e73c81b1ca56d057af91ec123f103f4be21b0d31450f0
+size 25583536

fold_1/model.chrombpnet_nobias.fold_1.ENCSR543YPH.tar ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:bf9c9185488ad941297d4435c0cd83b22e5f0afa05696eeb77659966451d9361
+size 26081280

fold_2/logs.models.fold_2.ENCSR543YPH/logfile.modelling.fold_2.ENCSR543YPH.args.json ADDED Viewed

	@@ -0,0 +1,23 @@

+{
+    "genome": "/scratch/groups/akundaje/anusri/chromatin_atlas/reference/hg38.genome.fa",
+    "bigwig": "/oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR543YPH//preprocessing/bigWigs/ENCSR543YPH.bigWig",
+    "peaks": "/scratch/groups/akundaje/anusri/chromatin_atlas/DNASE/ENCSR543YPH//chrombppnet_model_encsr880cub_bias_fold_2//filtered.peaks.bed",
+    "nonpeaks": "/scratch/groups/akundaje/anusri/chromatin_atlas/DNASE/ENCSR543YPH//chrombppnet_model_encsr880cub_bias_fold_2//filtered.nonpeaks.bed",
+    "output_prefix": "/scratch/groups/akundaje/anusri/chromatin_atlas/DNASE/ENCSR543YPH//chrombppnet_model_encsr880cub_bias_fold_2//chrombpnet",
+    "chr_fold_path": "/scratch/groups/akundaje/anusri/chromatin_atlas/splits/fold_2.json",
+    "trackables": [
+        "logcount_predictions_loss",
+        "loss",
+        "logits_profile_predictions_loss",
+        "val_logcount_predictions_loss",
+        "val_loss",
+        "val_logits_profile_predictions_loss"
+    ],
+    "epochs": 50,
+    "early_stop": 5,
+    "batch_size": 64,
+    "learning_rate": 0.001,
+    "params": "/scratch/groups/akundaje/anusri/chromatin_atlas/DNASE/ENCSR543YPH//chrombppnet_model_encsr880cub_bias_fold_2//chrombpnet_model_params.tsv",
+    "seed": 1234,
+    "architecture_from_file": "/home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/models/chrombpnet_with_bias_model.py"
+}

fold_2/logs.models.fold_2.ENCSR543YPH/logfile.modelling.fold_2.ENCSR543YPH.batch_loss.tsv ADDED Viewed

The diff for this file is too large to render. See raw diff

fold_2/logs.models.fold_2.ENCSR543YPH/logfile.modelling.fold_2.ENCSR543YPH.bias_formatting.stderr.txt ADDED Viewed

	@@ -0,0 +1,38 @@

+INFO:    underlay of /etc/localtime required more than 50 (88) bind mounts
+INFO:    underlay of /usr/bin/nvidia-smi required more than 50 (355) bind mounts
+2023-07-16 23:07:35.447585: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-07-16 23:07:37.748735: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set
+2023-07-16 23:07:37.751696: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1
+2023-07-16 23:07:38.311359: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties:
+pciBusID: 0000:c0:00.0 name: NVIDIA A100-SXM4-80GB computeCapability: 8.0
+coreClock: 1.41GHz coreCount: 108 deviceMemorySize: 79.15GiB deviceMemoryBandwidth: 1.85TiB/s
+2023-07-16 23:07:38.311414: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-07-16 23:07:38.329696: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2023-07-16 23:07:38.329748: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2023-07-16 23:07:38.339559: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
+2023-07-16 23:07:38.344132: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
+2023-07-16 23:07:38.359984: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
+2023-07-16 23:07:38.364116: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
+2023-07-16 23:07:38.365014: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+2023-07-16 23:07:38.379734: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
+2023-07-16 23:07:38.380138: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+2023-07-16 23:07:38.381323: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set
+2023-07-16 23:07:38.403955: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties:
+pciBusID: 0000:c0:00.0 name: NVIDIA A100-SXM4-80GB computeCapability: 8.0
+coreClock: 1.41GHz coreCount: 108 deviceMemorySize: 79.15GiB deviceMemoryBandwidth: 1.85TiB/s
+2023-07-16 23:07:38.404025: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-07-16 23:07:38.404064: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2023-07-16 23:07:38.404093: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2023-07-16 23:07:38.404119: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
+2023-07-16 23:07:38.404144: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
+2023-07-16 23:07:38.404168: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
+2023-07-16 23:07:38.404192: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
+2023-07-16 23:07:38.404217: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+2023-07-16 23:07:38.451623: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
+2023-07-16 23:07:38.454027: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-07-16 23:07:41.495725: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix:
+2023-07-16 23:07:41.495834: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267]      0
+2023-07-16 23:07:41.495849: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0:   N
+2023-07-16 23:07:41.502726: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 75650 MB memory) -> physical GPU (device: 0, name: NVIDIA A100-SXM4-80GB, pci bus id: 0000:c0:00.0, compute capability: 8.0)
+2023-07-16 23:07:42.743252: W tensorflow/python/util/util.cc:348] Sets are not currently considered sequences, but this may change in the future, so consider avoiding using them.

fold_2/logs.models.fold_2.ENCSR543YPH/logfile.modelling.fold_2.ENCSR543YPH.bias_formatting.stdout.txt ADDED Viewed

	@@ -0,0 +1 @@

+ singularity exec --nv /home/groups/akundaje/anusri/simg/tf-atlas_gcp-modeling.sif python get_new_tf_model_format.py -i /oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR543YPH//chrombppnet_model_encsr880cub_bias_fold_2/bias_model_scaled.h5 -o /oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR543YPH//chrombppnet_model_encsr880cub_bias_fold_2/new_model_formats/bias_model_scaled

fold_2/logs.models.fold_2.ENCSR543YPH/logfile.modelling.fold_2.ENCSR543YPH.chrombpnet.params.json ADDED Viewed

	@@ -0,0 +1,11 @@

+{
+    "counts_loss_weight": "4.0",
+    "filters": "512",
+    "n_dil_layers": "8",
+    "bias_model_path": "/scratch/groups/akundaje/anusri/chromatin_atlas/DNASE/ENCSR543YPH//chrombppnet_model_encsr880cub_bias_fold_2/bias_model_scaled.h5",
+    "inputlen": "2114",
+    "outputlen": "1000",
+    "max_jitter": "500",
+    "chr_fold_path": "/scratch/groups/akundaje/anusri/chromatin_atlas/splits/fold_2.json",
+    "negative_sampling_ratio": "0.1"
+}

fold_2/logs.models.fold_2.ENCSR543YPH/logfile.modelling.fold_2.ENCSR543YPH.chrombpnet_data_params.tsv ADDED Viewed

	@@ -0,0 +1,3 @@

+counts_sum_min_thresh	0.0
+counts_sum_max_thresh	873.0
+trainings_pts_post_thresh	175216

fold_2/logs.models.fold_2.ENCSR543YPH/logfile.modelling.fold_2.ENCSR543YPH.chrombpnet_formatting.stderr.txt ADDED Viewed

	@@ -0,0 +1,40 @@

+INFO:    underlay of /etc/localtime required more than 50 (88) bind mounts
+INFO:    underlay of /usr/bin/nvidia-smi required more than 50 (355) bind mounts
+2023-07-15 02:10:52.140326: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-07-15 02:10:55.121761: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set
+2023-07-15 02:10:55.127895: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1
+2023-07-15 02:10:55.166811: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties:
+pciBusID: 0000:83:00.0 name: Tesla P100-PCIE-16GB computeCapability: 6.0
+coreClock: 1.3285GHz coreCount: 56 deviceMemorySize: 15.89GiB deviceMemoryBandwidth: 681.88GiB/s
+2023-07-15 02:10:55.166909: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-07-15 02:10:55.198564: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2023-07-15 02:10:55.198702: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2023-07-15 02:10:55.213419: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
+2023-07-15 02:10:55.219868: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
+2023-07-15 02:10:55.243129: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
+2023-07-15 02:10:55.248868: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
+2023-07-15 02:10:55.250115: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+2023-07-15 02:10:55.252484: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
+2023-07-15 02:10:55.252865: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+2023-07-15 02:10:55.253849: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set
+2023-07-15 02:10:55.269734: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties:
+pciBusID: 0000:83:00.0 name: Tesla P100-PCIE-16GB computeCapability: 6.0
+coreClock: 1.3285GHz coreCount: 56 deviceMemorySize: 15.89GiB deviceMemoryBandwidth: 681.88GiB/s
+2023-07-15 02:10:55.269786: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-07-15 02:10:55.269826: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2023-07-15 02:10:55.269852: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2023-07-15 02:10:55.269877: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
+2023-07-15 02:10:55.269902: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
+2023-07-15 02:10:55.269926: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
+2023-07-15 02:10:55.269950: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
+2023-07-15 02:10:55.269986: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+2023-07-15 02:10:55.270894: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
+2023-07-15 02:10:55.272447: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-07-15 02:10:57.143804: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix:
+2023-07-15 02:10:57.143906: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267]      0
+2023-07-15 02:10:57.143924: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0:   N
+2023-07-15 02:10:57.146868: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 14957 MB memory) -> physical GPU (device: 0, name: Tesla P100-PCIE-16GB, pci bus id: 0000:83:00.0, compute capability: 6.0)
+2023-07-15 02:10:59.483739: W tensorflow/python/util/util.cc:348] Sets are not currently considered sequences, but this may change in the future, so consider avoiding using them.
+/opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/layers/core.py:1059: UserWarning:  is not loaded, but a Lambda layer uses it. It may cause errors.
+  , UserWarning)

fold_2/logs.models.fold_2.ENCSR543YPH/logfile.modelling.fold_2.ENCSR543YPH.chrombpnet_formatting.stdout.txt ADDED Viewed

	@@ -0,0 +1 @@

+ singularity exec --nv /home/groups/akundaje/anusri/simg/tf-atlas_gcp-modeling.sif python get_new_tf_model_format.py -i /oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR543YPH//chrombppnet_model_encsr880cub_bias_fold_2/chrombpnet.h5 -o /oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR543YPH//chrombppnet_model_encsr880cub_bias_fold_2/new_model_formats/chrombpnet

fold_2/logs.models.fold_2.ENCSR543YPH/logfile.modelling.fold_2.ENCSR543YPH.chrombpnet_model_params.tsv ADDED Viewed

	@@ -0,0 +1,9 @@

+counts_loss_weight	4.0
+filters	512
+n_dil_layers	8
+bias_model_path	/scratch/groups/akundaje/anusri/chromatin_atlas/DNASE/ENCSR543YPH//chrombppnet_model_encsr880cub_bias_fold_2/bias_model_scaled.h5
+inputlen	2114
+outputlen	1000
+max_jitter	500
+chr_fold_path	/scratch/groups/akundaje/anusri/chromatin_atlas/splits/fold_2.json
+negative_sampling_ratio	0.1

fold_2/logs.models.fold_2.ENCSR543YPH/logfile.modelling.fold_2.ENCSR543YPH.chrombpnet_no_bias_formatting.stderr.txt ADDED Viewed

	@@ -0,0 +1 @@

+ singularity exec --nv /home/groups/akundaje/anusri/simg/tf-atlas_gcp-modeling.sif python get_new_tf_model_format.py -i /oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR543YPH//chrombppnet_model_encsr880cub_bias_fold_2/chrombpnet_wo_bias.h5 -o /oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR543YPH//chrombppnet_model_encsr880cub_bias_fold_2/new_model_formats/chrombpnet_wo_bias

fold_2/logs.models.fold_2.ENCSR543YPH/logfile.modelling.fold_2.ENCSR543YPH.chrombpnet_no_bias_formatting.stdout.txt ADDED Viewed

	@@ -0,0 +1 @@

+ singularity exec --nv /home/groups/akundaje/anusri/simg/tf-atlas_gcp-modeling.sif python get_new_tf_model_format.py -i /oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR543YPH//chrombppnet_model_encsr880cub_bias_fold_2/chrombpnet_wo_bias.h5 -o /oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR543YPH//chrombppnet_model_encsr880cub_bias_fold_2/new_model_formats/chrombpnet_wo_bias

fold_2/logs.models.fold_2.ENCSR543YPH/logfile.modelling.fold_2.ENCSR543YPH.epoch_loss.csv ADDED Viewed

	@@ -0,0 +1,15 @@

+epoch,logcount_predictions_loss,logits_profile_predictions_loss,loss,val_logcount_predictions_loss,val_logits_profile_predictions_loss,val_loss
+0,2.546609401702881,217.68313598632812,227.8693084716797,0.7787268161773682,218.65565490722656,221.77049255371094
+1,0.8022686839103699,204.4232940673828,207.63259887695312,0.7655566930770874,213.34506225585938,216.4072723388672
+2,0.7345540523529053,200.96913146972656,203.9075469970703,0.6925560235977173,212.24774169921875,215.0181121826172
+3,0.6894159317016602,198.87734985351562,201.63526916503906,0.6129319071769714,211.69290161132812,214.1444549560547
+4,0.6634780168533325,197.58522033691406,200.23895263671875,0.5958423614501953,208.92355346679688,211.30691528320312
+5,0.6392917037010193,196.55210876464844,199.1094512939453,0.5904695391654968,209.49942016601562,211.8612823486328
+6,0.6120452284812927,195.63316345214844,198.08148193359375,0.5765258073806763,209.06443786621094,211.3704071044922
+7,0.5996060371398926,194.2079620361328,196.60655212402344,0.5488075613975525,209.30760192871094,211.50286865234375
+8,0.5408955812454224,191.82650756835938,193.9901580810547,0.5564015507698059,208.03501892089844,210.26065063476562
+9,0.5284738540649414,190.56497192382812,192.67901611328125,0.536838710308075,208.9392547607422,211.08653259277344
+10,0.5213044881820679,188.974365234375,191.05966186523438,0.5298101902008057,208.7268829345703,210.84620666503906
+11,0.5115966796875,188.03500366210938,190.08148193359375,0.522258996963501,208.85382080078125,210.9427947998047
+12,0.48411017656326294,186.51986694335938,188.4563446044922,0.5244764685630798,209.2521209716797,211.34996032714844
+13,0.4746929109096527,185.52134704589844,187.42002868652344,0.5278152823448181,209.30325317382812,211.4146270751953

fold_2/logs.models.fold_2.ENCSR543YPH/logfile.modelling.fold_2.ENCSR543YPH.stderr.txt ADDED Viewed

	@@ -0,0 +1,336 @@

+INFO:    underlay of /etc/localtime required more than 50 (88) bind mounts
+INFO:    underlay of /usr/bin/nvidia-smi required more than 50 (355) bind mounts
+INFO:    underlay of /etc/localtime required more than 50 (88) bind mounts
+INFO:    underlay of /usr/bin/nvidia-smi required more than 50 (355) bind mounts
+2022-10-15 01:55:59.626987: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2022-10-15 02:06:16.267219: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set
+2022-10-15 02:06:16.272549: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1
+2022-10-15 02:06:16.338011: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties:
+pciBusID: 0000:09:00.0 name: NVIDIA TITAN V computeCapability: 7.0
+coreClock: 1.455GHz coreCount: 80 deviceMemorySize: 11.78GiB deviceMemoryBandwidth: 607.97GiB/s
+2022-10-15 02:06:16.338145: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2022-10-15 02:06:16.370087: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2022-10-15 02:06:16.370269: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2022-10-15 02:06:16.387094: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
+2022-10-15 02:06:16.394650: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
+2022-10-15 02:06:16.423856: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
+2022-10-15 02:06:16.431003: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
+2022-10-15 02:06:16.432784: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+2022-10-15 02:06:16.435611: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
+2022-10-15 02:06:16.436135: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+2022-10-15 02:06:16.437272: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set
+2022-10-15 02:06:16.438204: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties:
+pciBusID: 0000:09:00.0 name: NVIDIA TITAN V computeCapability: 7.0
+coreClock: 1.455GHz coreCount: 80 deviceMemorySize: 11.78GiB deviceMemoryBandwidth: 607.97GiB/s
+2022-10-15 02:06:16.438279: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2022-10-15 02:06:16.438331: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2022-10-15 02:06:16.438362: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2022-10-15 02:06:16.438388: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
+2022-10-15 02:06:16.438415: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
+2022-10-15 02:06:16.438441: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
+2022-10-15 02:06:16.438467: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
+2022-10-15 02:06:16.438494: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+2022-10-15 02:06:16.439438: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
+2022-10-15 02:06:16.441205: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2022-10-15 02:06:18.680129: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix:
+2022-10-15 02:06:18.680244: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267]      0
+2022-10-15 02:06:18.680266: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0:   N
+2022-10-15 02:06:18.685655: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10912 MB memory) -> physical GPU (device: 0, name: NVIDIA TITAN V, pci bus id: 0000:09:00.0, compute capability: 7.0)
+2022-10-15 02:06:20.623604: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:116] None of the MLIR optimization passes are enabled (registered 2)
+2022-10-15 02:06:20.642499: I tensorflow/core/platform/profile_utils/cpu_utils.cc:112] CPU Frequency: 2400085000 Hz
+2022-10-15 02:06:20.929781: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2022-10-15 02:06:22.328557: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2022-10-15 02:06:22.340100: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+2022-10-15 02:07:15.616981: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2022-10-15 02:07:18.120702: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set
+2022-10-15 02:07:18.121940: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1
+2022-10-15 02:07:18.179485: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties:
+pciBusID: 0000:09:00.0 name: NVIDIA TITAN V computeCapability: 7.0
+coreClock: 1.455GHz coreCount: 80 deviceMemorySize: 11.78GiB deviceMemoryBandwidth: 607.97GiB/s
+2022-10-15 02:07:18.179594: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2022-10-15 02:07:18.183232: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2022-10-15 02:07:18.183351: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2022-10-15 02:07:18.184949: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
+2022-10-15 02:07:18.185376: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
+2022-10-15 02:07:18.188846: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
+2022-10-15 02:07:18.189776: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
+2022-10-15 02:07:18.190193: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+2022-10-15 02:07:18.191232: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
+2022-10-15 02:07:18.191617: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+2022-10-15 02:07:18.191721: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set
+2022-10-15 02:07:18.192256: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties:
+pciBusID: 0000:09:00.0 name: NVIDIA TITAN V computeCapability: 7.0
+coreClock: 1.455GHz coreCount: 80 deviceMemorySize: 11.78GiB deviceMemoryBandwidth: 607.97GiB/s
+2022-10-15 02:07:18.192290: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2022-10-15 02:07:18.192326: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2022-10-15 02:07:18.192353: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2022-10-15 02:07:18.192377: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
+2022-10-15 02:07:18.192401: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
+2022-10-15 02:07:18.192425: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
+2022-10-15 02:07:18.192449: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
+2022-10-15 02:07:18.192473: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+2022-10-15 02:07:18.193359: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
+2022-10-15 02:07:18.193403: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2022-10-15 02:07:18.901427: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix:
+2022-10-15 02:07:18.901554: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267]      0
+2022-10-15 02:07:18.901575: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0:   N
+2022-10-15 02:07:18.904060: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10912 MB memory) -> physical GPU (device: 0, name: NVIDIA TITAN V, pci bus id: 0000:09:00.0, compute capability: 7.0)
+2022-10-15 02:16:55.815972: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:116] None of the MLIR optimization passes are enabled (registered 2)
+2022-10-15 02:16:55.816581: I tensorflow/core/platform/profile_utils/cpu_utils.cc:112] CPU Frequency: 2400085000 Hz
+2022-10-15 02:16:58.059686: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2022-10-15 02:16:58.438800: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2022-10-15 02:16:58.462994: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 0.2106s vs `on_train_batch_end` time: 0.2960s). Check your callbacks.
+2022-10-15 08:07:56.729925: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2022-10-15 08:08:00.319352: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set
+2022-10-15 08:08:00.320706: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1
+2022-10-15 08:08:00.373335: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties:
+pciBusID: 0000:09:00.0 name: NVIDIA TITAN V computeCapability: 7.0
+coreClock: 1.455GHz coreCount: 80 deviceMemorySize: 11.78GiB deviceMemoryBandwidth: 607.97GiB/s
+2022-10-15 08:08:00.373447: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2022-10-15 08:08:00.377116: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2022-10-15 08:08:00.377299: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2022-10-15 08:08:00.378913: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
+2022-10-15 08:08:00.379302: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
+2022-10-15 08:08:00.382710: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
+2022-10-15 08:08:00.383702: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
+2022-10-15 08:08:00.384124: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+2022-10-15 08:08:00.385174: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
+2022-10-15 08:08:00.385601: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+2022-10-15 08:08:00.385715: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set
+2022-10-15 08:08:00.388375: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties:
+pciBusID: 0000:09:00.0 name: NVIDIA TITAN V computeCapability: 7.0
+coreClock: 1.455GHz coreCount: 80 deviceMemorySize: 11.78GiB deviceMemoryBandwidth: 607.97GiB/s
+2022-10-15 08:08:00.388446: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2022-10-15 08:08:00.388515: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2022-10-15 08:08:00.388546: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2022-10-15 08:08:00.388575: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
+2022-10-15 08:08:00.388604: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
+2022-10-15 08:08:00.388632: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
+2022-10-15 08:08:00.388660: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
+2022-10-15 08:08:00.388689: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+2022-10-15 08:08:00.389596: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
+2022-10-15 08:08:00.389647: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2022-10-15 08:08:01.102732: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix:
+2022-10-15 08:08:01.102843: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267]      0
+2022-10-15 08:08:01.102863: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0:   N
+2022-10-15 08:08:01.106101: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10912 MB memory) -> physical GPU (device: 0, name: NVIDIA TITAN V, pci bus id: 0000:09:00.0, compute capability: 7.0)
+2022-10-15 08:09:41.696266: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:116] None of the MLIR optimization passes are enabled (registered 2)
+2022-10-15 08:09:41.700779: I tensorflow/core/platform/profile_utils/cpu_utils.cc:112] CPU Frequency: 2400085000 Hz
+2022-10-15 08:09:41.824847: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2022-10-15 08:09:42.123076: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2022-10-15 08:09:42.125669: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+/opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/layers/core.py:1059: UserWarning:  is not loaded, but a Lambda layer uses it. It may cause errors.
+  , UserWarning)
+/home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:69: RuntimeWarning: invalid value encountered in true_divide
+  cur_jsd=jensenshannon(true_counts[idx,:]/np.nansum(true_counts[idx,:]),pred_probs[idx,:])
+/home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/utils/metrics_utils.py:196: RuntimeWarning: invalid value encountered in true_divide
+  profile_prob = profile / np.sum(profile)
+/home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:78: RuntimeWarning: invalid value encountered in true_divide
+  shuffled_labels_prob=shuffled_labels/np.nansum(shuffled_labels)
+/home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:88: RuntimeWarning: invalid value encountered in true_divide
+  curr_jsd_rnd=jensenshannon(true_counts[idx,:]/np.nansum(true_counts[idx,:]),shuffled_labels_prob)
+No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.
+findfont: Font family ['normal'] not found. Falling back to DejaVu Sans.
+findfont: Font family ['normal'] not found. Falling back to DejaVu Sans.
+findfont: Font family ['normal'] not found. Falling back to DejaVu Sans.
+No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.
+No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.
+2022-10-15 08:14:35.961009: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2022-10-15 08:14:39.052512: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set
+2022-10-15 08:14:39.053695: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1
+2022-10-15 08:14:39.108626: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties:
+pciBusID: 0000:09:00.0 name: NVIDIA TITAN V computeCapability: 7.0
+coreClock: 1.455GHz coreCount: 80 deviceMemorySize: 11.78GiB deviceMemoryBandwidth: 607.97GiB/s
+2022-10-15 08:14:39.108693: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2022-10-15 08:14:39.112006: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2022-10-15 08:14:39.112071: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2022-10-15 08:14:39.113564: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
+2022-10-15 08:14:39.113926: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
+2022-10-15 08:14:39.117093: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
+2022-10-15 08:14:39.117945: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
+2022-10-15 08:14:39.118384: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+2022-10-15 08:14:39.123489: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
+2022-10-15 08:14:39.123846: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+2022-10-15 08:14:39.123938: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set
+2022-10-15 08:14:39.124431: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties:
+pciBusID: 0000:09:00.0 name: NVIDIA TITAN V computeCapability: 7.0
+coreClock: 1.455GHz coreCount: 80 deviceMemorySize: 11.78GiB deviceMemoryBandwidth: 607.97GiB/s
+2022-10-15 08:14:39.124466: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2022-10-15 08:14:39.124496: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2022-10-15 08:14:39.124523: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2022-10-15 08:14:39.124550: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
+2022-10-15 08:14:39.124577: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
+2022-10-15 08:14:39.124602: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
+2022-10-15 08:14:39.124628: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
+2022-10-15 08:14:39.124654: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+2022-10-15 08:14:39.127816: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
+2022-10-15 08:14:39.127870: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2022-10-15 08:14:39.820345: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix:
+2022-10-15 08:14:40.744235: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267]      0
+2022-10-15 08:14:40.744356: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0:   N
+2022-10-15 08:14:40.751039: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10912 MB memory) -> physical GPU (device: 0, name: NVIDIA TITAN V, pci bus id: 0000:09:00.0, compute capability: 7.0)
+WARNING:tensorflow:No training configuration found in the save file, so the model was *not* compiled. Compile it manually.
+2022-10-15 08:16:17.286055: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:116] None of the MLIR optimization passes are enabled (registered 2)
+2022-10-15 08:16:17.289035: I tensorflow/core/platform/profile_utils/cpu_utils.cc:112] CPU Frequency: 2400085000 Hz
+2022-10-15 08:16:17.363547: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2022-10-15 08:16:17.646175: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2022-10-15 08:16:17.647963: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+/home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:69: RuntimeWarning: invalid value encountered in true_divide
+  cur_jsd=jensenshannon(true_counts[idx,:]/np.nansum(true_counts[idx,:]),pred_probs[idx,:])
+/home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/utils/metrics_utils.py:196: RuntimeWarning: invalid value encountered in true_divide
+  profile_prob = profile / np.sum(profile)
+/home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:78: RuntimeWarning: invalid value encountered in true_divide
+  shuffled_labels_prob=shuffled_labels/np.nansum(shuffled_labels)
+/home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:88: RuntimeWarning: invalid value encountered in true_divide
+  curr_jsd_rnd=jensenshannon(true_counts[idx,:]/np.nansum(true_counts[idx,:]),shuffled_labels_prob)
+No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.
+findfont: Font family ['normal'] not found. Falling back to DejaVu Sans.
+findfont: Font family ['normal'] not found. Falling back to DejaVu Sans.
+findfont: Font family ['normal'] not found. Falling back to DejaVu Sans.
+No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.
+No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.
+2022-10-15 08:20:58.065612: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2022-10-15 08:21:01.342750: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set
+2022-10-15 08:21:01.343926: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1
+2022-10-15 08:21:01.398102: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties:
+pciBusID: 0000:09:00.0 name: NVIDIA TITAN V computeCapability: 7.0
+coreClock: 1.455GHz coreCount: 80 deviceMemorySize: 11.78GiB deviceMemoryBandwidth: 607.97GiB/s
+2022-10-15 08:21:01.398208: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2022-10-15 08:21:01.401799: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2022-10-15 08:21:01.401864: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2022-10-15 08:21:01.403323: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
+2022-10-15 08:21:01.403671: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
+2022-10-15 08:21:01.407052: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
+2022-10-15 08:21:01.407874: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
+2022-10-15 08:21:01.408264: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+2022-10-15 08:21:01.409197: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
+2022-10-15 08:21:01.409599: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+2022-10-15 08:21:01.409719: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set
+2022-10-15 08:21:01.410193: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties:
+pciBusID: 0000:09:00.0 name: NVIDIA TITAN V computeCapability: 7.0
+coreClock: 1.455GHz coreCount: 80 deviceMemorySize: 11.78GiB deviceMemoryBandwidth: 607.97GiB/s
+2022-10-15 08:21:01.410227: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2022-10-15 08:21:01.410278: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2022-10-15 08:21:01.410314: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2022-10-15 08:21:01.410355: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
+2022-10-15 08:21:01.410381: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
+2022-10-15 08:21:01.410407: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
+2022-10-15 08:21:01.410433: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
+2022-10-15 08:21:01.410459: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+2022-10-15 08:21:01.413321: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
+2022-10-15 08:21:01.413375: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2022-10-15 08:21:02.116504: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix:
+2022-10-15 08:21:02.116610: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267]      0
+2022-10-15 08:21:02.116643: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0:   N
+2022-10-15 08:21:02.118232: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10912 MB memory) -> physical GPU (device: 0, name: NVIDIA TITAN V, pci bus id: 0000:09:00.0, compute capability: 7.0)
+2022-10-15 08:22:38.700051: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:116] None of the MLIR optimization passes are enabled (registered 2)
+2022-10-15 08:22:38.702225: I tensorflow/core/platform/profile_utils/cpu_utils.cc:112] CPU Frequency: 2400085000 Hz
+2022-10-15 08:22:38.751442: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2022-10-15 08:22:39.043455: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2022-10-15 08:22:39.045177: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+/home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:69: RuntimeWarning: invalid value encountered in true_divide
+  cur_jsd=jensenshannon(true_counts[idx,:]/np.nansum(true_counts[idx,:]),pred_probs[idx,:])
+/home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/utils/metrics_utils.py:196: RuntimeWarning: invalid value encountered in true_divide
+  profile_prob = profile / np.sum(profile)
+/home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:78: RuntimeWarning: invalid value encountered in true_divide
+  shuffled_labels_prob=shuffled_labels/np.nansum(shuffled_labels)
+/home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:88: RuntimeWarning: invalid value encountered in true_divide
+  curr_jsd_rnd=jensenshannon(true_counts[idx,:]/np.nansum(true_counts[idx,:]),shuffled_labels_prob)
+No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.
+findfont: Font family ['normal'] not found. Falling back to DejaVu Sans.
+findfont: Font family ['normal'] not found. Falling back to DejaVu Sans.
+findfont: Font family ['normal'] not found. Falling back to DejaVu Sans.
+No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.
+No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.
+2022-10-15 08:24:05.615241: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2022-10-15 08:24:07.219332: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set
+2022-10-15 08:24:07.220493: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1
+2022-10-15 08:24:07.269691: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties:
+pciBusID: 0000:09:00.0 name: NVIDIA TITAN V computeCapability: 7.0
+coreClock: 1.455GHz coreCount: 80 deviceMemorySize: 11.78GiB deviceMemoryBandwidth: 607.97GiB/s
+2022-10-15 08:24:07.269802: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2022-10-15 08:24:07.273194: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2022-10-15 08:24:07.273262: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2022-10-15 08:24:07.274811: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
+2022-10-15 08:24:07.275170: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
+2022-10-15 08:24:07.278453: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
+2022-10-15 08:24:07.279273: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
+2022-10-15 08:24:07.279788: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+2022-10-15 08:24:07.281453: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
+2022-10-15 08:24:07.281801: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+2022-10-15 08:24:07.281890: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set
+2022-10-15 08:24:07.282375: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties:
+pciBusID: 0000:09:00.0 name: NVIDIA TITAN V computeCapability: 7.0
+coreClock: 1.455GHz coreCount: 80 deviceMemorySize: 11.78GiB deviceMemoryBandwidth: 607.97GiB/s
+2022-10-15 08:24:07.282415: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2022-10-15 08:24:07.282443: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2022-10-15 08:24:07.282469: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2022-10-15 08:24:07.282493: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
+2022-10-15 08:24:07.282518: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
+2022-10-15 08:24:07.282543: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
+2022-10-15 08:24:07.282567: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
+2022-10-15 08:24:07.282592: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+2022-10-15 08:24:07.283403: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
+2022-10-15 08:24:07.283472: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2022-10-15 08:24:07.982636: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix:
+2022-10-15 08:24:07.982743: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267]      0
+2022-10-15 08:24:07.982762: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0:   N
+2022-10-15 08:24:07.984351: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10912 MB memory) -> physical GPU (device: 0, name: NVIDIA TITAN V, pci bus id: 0000:09:00.0, compute capability: 7.0)
+WARNING:tensorflow:No training configuration found in the save file, so the model was *not* compiled. Compile it manually.
+2022-10-15 08:24:25.067210: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:116] None of the MLIR optimization passes are enabled (registered 2)
+2022-10-15 08:24:25.067846: I tensorflow/core/platform/profile_utils/cpu_utils.cc:112] CPU Frequency: 2400085000 Hz
+2022-10-15 08:24:25.342505: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2022-10-15 08:24:25.706436: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2022-10-15 08:24:25.708452: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+2022-10-15 08:24:30.994959: W tensorflow/core/common_runtime/bfc_allocator.cc:248] Allocator (GPU_0_bfc) ran out of memory trying to allocate 3.96GiB with freed_by_count=0. The caller indicates that this is not a failure, but may mean that there could be performance gains if more memory were available.
+2022-10-15 08:24:30.995498: W tensorflow/core/common_runtime/bfc_allocator.cc:248] Allocator (GPU_0_bfc) ran out of memory trying to allocate 3.96GiB with freed_by_count=0. The caller indicates that this is not a failure, but may mean that there could be performance gains if more memory were available.
+2022-10-15 08:24:31.492299: W tensorflow/core/common_runtime/bfc_allocator.cc:248] Allocator (GPU_0_bfc) ran out of memory trying to allocate 3.77GiB with freed_by_count=0. The caller indicates that this is not a failure, but may mean that there could be performance gains if more memory were available.
+2022-10-15 08:24:31.492822: W tensorflow/core/common_runtime/bfc_allocator.cc:248] Allocator (GPU_0_bfc) ran out of memory trying to allocate 3.77GiB with freed_by_count=0. The caller indicates that this is not a failure, but may mean that there could be performance gains if more memory were available.
+mkdir: cannot create directory ‘/scratch/groups/akundaje/anusri/chromatin_atlas/DNASE/ENCSR543YPH//chrombppnet_model_encsr880cub_bias_fold_2//footprints’: File exists
+2022-10-15 08:28:05.210082: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2022-10-15 08:28:06.813905: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set
+2022-10-15 08:28:06.815099: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1
+2022-10-15 08:28:06.879591: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties:
+pciBusID: 0000:09:00.0 name: NVIDIA TITAN V computeCapability: 7.0
+coreClock: 1.455GHz coreCount: 80 deviceMemorySize: 11.78GiB deviceMemoryBandwidth: 607.97GiB/s
+2022-10-15 08:28:06.879721: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2022-10-15 08:28:06.883099: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2022-10-15 08:28:06.883167: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2022-10-15 08:28:06.884747: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
+2022-10-15 08:28:06.885099: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
+2022-10-15 08:28:06.888527: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
+2022-10-15 08:28:06.889352: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
+2022-10-15 08:28:06.889772: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+2022-10-15 08:28:06.890693: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
+2022-10-15 08:28:06.891039: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+2022-10-15 08:28:06.891123: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set
+2022-10-15 08:28:06.891717: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties:
+pciBusID: 0000:09:00.0 name: NVIDIA TITAN V computeCapability: 7.0
+coreClock: 1.455GHz coreCount: 80 deviceMemorySize: 11.78GiB deviceMemoryBandwidth: 607.97GiB/s
+2022-10-15 08:28:06.891762: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2022-10-15 08:28:06.891795: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2022-10-15 08:28:06.891824: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2022-10-15 08:28:06.891853: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
+2022-10-15 08:28:06.891882: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
+2022-10-15 08:28:06.891910: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
+2022-10-15 08:28:06.891939: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
+2022-10-15 08:28:06.891968: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+2022-10-15 08:28:06.897076: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
+2022-10-15 08:28:06.897139: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2022-10-15 08:28:07.565117: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix:
+2022-10-15 08:28:07.565238: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267]      0
+2022-10-15 08:28:07.565257: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0:   N
+2022-10-15 08:28:07.567540: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10912 MB memory) -> physical GPU (device: 0, name: NVIDIA TITAN V, pci bus id: 0000:09:00.0, compute capability: 7.0)
+2022-10-15 08:28:25.790426: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:116] None of the MLIR optimization passes are enabled (registered 2)
+2022-10-15 08:28:25.791043: I tensorflow/core/platform/profile_utils/cpu_utils.cc:112] CPU Frequency: 2400085000 Hz
+2022-10-15 08:28:25.978425: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2022-10-15 08:28:26.332301: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2022-10-15 08:28:26.334282: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8

fold_2/logs.models.fold_2.ENCSR543YPH/logfile.modelling.fold_2.ENCSR543YPH.stdout.txt ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:a3bc4d18095b1481760ae2bc4bb1921dcd0b0601a7c3953a818ecc00cdee8fac
+size 10951833

fold_2/logs.models.fold_2.ENCSR543YPH/logfile.modelling.fold_2.ENCSR543YPH.stdout_v1.txt ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:38be7107a468bc16f3f1c1403fe7fc6fa07b4b74f51ed9cc152869df116e86fd
+size 10950160

fold_2/model.bias_scaled.fold_2.ENCSR543YPH.h5 ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:fe96a8097dffa5a2d8c06f4686d5379cd754b622cea57830aa397f199cc8934a
+size 2688440

fold_2/model.bias_scaled.fold_2.ENCSR543YPH.tar ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:f3aacde3079ed2a11784bbee7c319d8c9ddbcfd076fee56a0c8745ac7001efa6
+size 1198080

fold_2/model.chrombpnet.fold_2.ENCSR543YPH.h5 ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:68023b3835fa8eb9bd362ccb42f1eeca5be4a133b6a94ae7a88fa97f44fcab18
+size 26447928

fold_2/model.chrombpnet.fold_2.ENCSR543YPH.tar ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:74cfa215b5f701572ed70acd5d553b4d46c7736046d9424f612c26f4a60e33a6
+size 27607040

fold_2/model.chrombpnet_nobias.fold_2.ENCSR543YPH.h5 ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:18f9ba0e6f52f313a827c8c409cc8ae7a08a1df4463a090d75d9cce4a4938aef
+size 25583536

fold_2/model.chrombpnet_nobias.fold_2.ENCSR543YPH.tar ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:8de424ac6a8c0b42909b1cbb33a05eebbc8e65d4fbefd9e8558a54f7e0ad8f4d
+size 26081280

fold_3/logs.models.fold_3.ENCSR543YPH/logfile.modelling.fold_3.ENCSR543YPH.stdout.txt ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:9402acbf08257cb4af53f82888ed69a02c4c9f8d3220cef371909399825f3349
+size 11949140

fold_3/logs.models.fold_3.ENCSR543YPH/logfile.modelling.fold_3.ENCSR543YPH.stdout_v1.txt ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:f0cbbb8d051797f6251f1cb46dbc0546cef97ec0450ab5230b8292180b90ce7a
+size 11947467

fold_3/model.bias_scaled.fold_3.ENCSR543YPH.h5 ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:ddea920e569df6d5fa6e7db9a9b5e4d1a31ec98932a2ca74473d291302552a54
+size 2688440

fold_3/model.bias_scaled.fold_3.ENCSR543YPH.tar ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:700e09f66d52a6265829e6bfa3c47c295fc0422ca00e9c310d9694e03881b039
+size 1198080

fold_3/model.chrombpnet.fold_3.ENCSR543YPH.h5 ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:6a4c9c5a8a2e0e40f9dcf28debc6b4e456207e49536dc6d77f1980302b866953
+size 26447928

fold_3/model.chrombpnet.fold_3.ENCSR543YPH.tar ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:915e99d02ca7287c6dcd2e642929a154ee380edfc12982be684d1e6d7b02b1d7
+size 27607040

fold_3/model.chrombpnet_nobias.fold_3.ENCSR543YPH.h5 ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:046829210269250332f4cd2ccbf09b25ca891d49159b172967840bbcd6a79b3f
+size 25583536