Merge branch 'test' of ssh://gitlab.cern.ch:7999/eneren/pytorchjob into test

73829ffe · Engin Eren · cc4df23a · 0ea555f9 · 73829ffe · 73829ffe
Commit 73829ffe authored 2 years ago by Engin Eren
--- a/interactive/eval.ipynb
+++ b/interactive/eval.ipynb
@@ -21,57 +21,96 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 4,
   "metadata": {},
-   "outputs": [
-    {
-     "ename": "OSError",
-     "evalue": "Unable to open file (unable to open file: name = '/eos/user/e/eneren/run_prod20k_40GeVp2/pion-shower_40.hdf5', errno = 13, error message = 'Permission denied', flags = 0, o_flags = 0)",
-     "output_type": "error",
-     "traceback": [
-      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
-      "\u001b[0;31mOSError\u001b[0m                                   Traceback (most recent call last)",
-      "\u001b[0;32m<ipython-input-3-e9542526b0da>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m## G4\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mh5py\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mFile\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'/eos/user/e/eneren/run_prod20k_40GeVp2/pion-shower_40.hdf5'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'r'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
-      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/h5py/_hl/files.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, name, mode, driver, libver, userblock_size, swmr, rdcc_nslots, rdcc_nbytes, rdcc_w0, track_order, **kwds)\u001b[0m\n\u001b[1;32m    406\u001b[0m                 fid = make_fid(name, mode, userblock_size,\n\u001b[1;32m    407\u001b[0m                                \u001b[0mfapl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfcpl\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmake_fcpl\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrack_order\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtrack_order\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 408\u001b[0;31m                                swmr=swmr)\n\u001b[0m\u001b[1;32m    409\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    410\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlibver\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtuple\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
-      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/h5py/_hl/files.py\u001b[0m in \u001b[0;36mmake_fid\u001b[0;34m(name, mode, userblock_size, fapl, fcpl, swmr)\u001b[0m\n\u001b[1;32m    171\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mswmr\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mswmr_support\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    172\u001b[0m             \u001b[0mflags\u001b[0m \u001b[0;34m|=\u001b[0m \u001b[0mh5f\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mACC_SWMR_READ\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 173\u001b[0;31m         \u001b[0mfid\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mh5f\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mflags\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfapl\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfapl\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    174\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0mmode\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'r+'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    175\u001b[0m         \u001b[0mfid\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mh5f\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mh5f\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mACC_RDWR\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfapl\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfapl\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
-      "\u001b[0;32mh5py/_objects.pyx\u001b[0m in \u001b[0;36mh5py._objects.with_phil.wrapper\u001b[0;34m()\u001b[0m\n",
-      "\u001b[0;32mh5py/_objects.pyx\u001b[0m in \u001b[0;36mh5py._objects.with_phil.wrapper\u001b[0;34m()\u001b[0m\n",
-      "\u001b[0;32mh5py/h5f.pyx\u001b[0m in \u001b[0;36mh5py.h5f.open\u001b[0;34m()\u001b[0m\n",
-      "\u001b[0;31mOSError\u001b[0m: Unable to open file (unable to open file: name = '/eos/user/e/eneren/run_prod20k_40GeVp2/pion-shower_40.hdf5', errno = 13, error message = 'Permission denied', flags = 0, o_flags = 0)"
-     ]
-    }
-   ],
+   "outputs": [],
   "source": [
    "## G4 \n",
-    "f = h5py.File('/eos/user/e/eneren/run_prod20k_40GeVp2/pion-shower_40.hdf5', 'r')\n"
+    "f40 = h5py.File('/eos/user/e/eneren/run_prod20k_40GeVp2/pion-shower_40.hdf5', 'r')\n",
+    "f50 = h5py.File('/eos/user/e/eneren/scratch/50GeV75k.hdf5', 'r')"
   ]
  },
  {
   "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
-    "showers = f['ecal/layers'][:]"
+    "showers50 = f50['ecal/layers'][:1000]\n",
+    "showers40 = f40['ecal/layers'][:1000]"
   ]
  },
  {
   "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 21,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "showers  = {\n",
+    "    '50': showers50,\n",
+    "    '40': showers40\n",
+    "}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
-     "ename": "PermissionError",
-     "evalue": "[Errno 13] Permission denied: '/eos/user/e/eneren/experiments/wganv1_generator_60.pt'",
-     "output_type": "error",
-     "traceback": [
-      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
-      "\u001b[0;31mPermissionError\u001b[0m                           Traceback (most recent call last)",
-      "\u001b[0;32m<ipython-input-4-4d3890daf5c7>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0meph\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m60\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m63\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 10\u001b[0;31m     \u001b[0mgen_checkpoint\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'/eos/user/e/eneren/experiments/'\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mexp\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m\"_generator_\"\u001b[0m\u001b[0;34m+\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0meph\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m\".pt\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmap_location\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'cpu'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     11\u001b[0m     \u001b[0mmGen\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload_state_dict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgen_checkpoint\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'model_state_dict'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m     \u001b[0mmGen\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0meval\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
-      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/serialization.py\u001b[0m in \u001b[0;36mload\u001b[0;34m(f, map_location, pickle_module, **pickle_load_args)\u001b[0m\n\u001b[1;32m    577\u001b[0m         \u001b[0mpickle_load_args\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'encoding'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m'utf-8'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    578\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 579\u001b[0;31m     \u001b[0;32mwith\u001b[0m \u001b[0m_open_file_like\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'rb'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mopened_file\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    580\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0m_is_zipfile\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mopened_file\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    581\u001b[0m             \u001b[0;31m# The zipfile reader is going to advance the current file position.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
-      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/serialization.py\u001b[0m in \u001b[0;36m_open_file_like\u001b[0;34m(name_or_buffer, mode)\u001b[0m\n\u001b[1;32m    228\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_open_file_like\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname_or_buffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    229\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0m_is_path\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname_or_buffer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 230\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0m_open_file\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname_or_buffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    231\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    232\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;34m'w'\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
-      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/serialization.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, name, mode)\u001b[0m\n\u001b[1;32m    209\u001b[0m \u001b[0;32mclass\u001b[0m \u001b[0m_open_file\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_opener\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    210\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__init__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 211\u001b[0;31m         \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_open_file\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__init__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    212\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    213\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__exit__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
-      "\u001b[0;31mPermissionError\u001b[0m: [Errno 13] Permission denied: '/eos/user/e/eneren/experiments/wganv1_generator_60.pt'"
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Epoch 251:  [0.3804679251946886, 0.34817149733098474]\n",
+      "Epoch 252:  [0.46051294878083926, 0.338569981047804]\n",
+      "Epoch 253:  [0.49756609463285095, 0.28872701740420126]\n",
+      "Epoch 254:  [0.5294431896884374, 0.33102840750614887]\n",
+      "Epoch 255:  [0.5104405441541784, 0.32231291300401893]\n",
+      "Epoch 256:  [0.48537347466233505, 0.320526721042005]\n",
+      "Epoch 257:  [0.44430318353292403, 0.33754094942412155]\n",
+      "Epoch 258:  [0.41426411913761607, 0.3383523881572018]\n",
+      "Epoch 259:  [0.4406027102881503, 0.3343011917750107]\n",
+      "Epoch 260:  [0.396682637265716, 0.3473503311876263]\n",
+      "Epoch 261:  [0.40394391363113635, 0.36565837165899845]\n",
+      "Epoch 262:  [0.42483844096641477, 0.32469973222063503]\n",
+      "Epoch 263:  [0.4019302446826285, 0.36214573291228813]\n",
+      "Epoch 264:  [0.4126153084092838, 0.38613505292409905]\n",
+      "Epoch 265:  [0.4165755030129357, 0.3519977789394489]\n",
+      "Epoch 266:  [0.3913068204960252, 0.38645762498974795]\n",
+      "Epoch 267:  [0.39439823796062096, 0.3708618579692532]\n",
+      "Epoch 268:  [0.40663386822619313, 0.37241345442978496]\n",
+      "Epoch 269:  [0.3683487445416732, 0.41097496954033574]\n",
+      "Epoch 270:  [0.3931176707467255, 0.36655001291434963]\n",
+      "Epoch 271:  [0.36265554202525746, 0.38067712153366035]\n",
+      "Epoch 272:  [0.36238044571407557, 0.3891110817723188]\n",
+      "Epoch 273:  [0.3443907936692471, 0.41633865859471425]\n",
+      "Epoch 274:  [0.3754340116921492, 0.3977884345978468]\n",
+      "Epoch 275:  [0.3552435525173964, 0.4127042666470706]\n",
+      "Epoch 276:  [0.3043411019678306, 0.4198835584501347]\n",
+      "Epoch 277:  [0.34320331541360355, 0.3928834088206806]\n",
+      "Epoch 278:  [0.31577926421268404, 0.4058244392791663]\n",
+      "Epoch 279:  [0.3009843850678768, 0.4373018678372319]\n",
+      "Epoch 280:  [0.3273260324046915, 0.4398480157854807]\n",
+      "Epoch 281:  [0.30707030715104516, 0.4339921708642802]\n",
+      "Epoch 282:  [0.2915953909635101, 0.4176873424583225]\n",
+      "Epoch 283:  [0.2924977435537306, 0.43666767755248453]\n",
+      "Epoch 284:  [0.2919643384290543, 0.45133477022104457]\n",
+      "Epoch 285:  [0.2947872964957196, 0.449896095813128]\n",
+      "Epoch 286:  [0.2897446726869525, 0.48724052437850385]\n",
+      "Epoch 287:  [0.31118635032181113, 0.4325288883457097]\n",
+      "Epoch 288:  [0.2696458856150868, 0.4629579633785893]\n",
+      "Epoch 289:  [0.28041806669044533, 0.46011915244113094]\n",
+      "Epoch 290:  [0.3074237951720241, 0.46164573769828265]\n",
+      "Epoch 291:  [0.28446450303954507, 0.463811909399871]\n",
+      "Epoch 292:  [0.2849400618724373, 0.4441966401101374]\n",
+      "Epoch 293:  [0.2826640112838286, 0.4795805721176796]\n",
+      "Epoch 294:  [0.2825341470608504, 0.4795195996538825]\n",
+      "Epoch 295:  [0.2805225945655388, 0.45946307781001905]\n",
+      "Epoch 296:  [0.28991007237838295, 0.46572437861315613]\n",
+      "Epoch 297:  [0.2668600287706407, 0.5065609989888424]\n",
+      "Epoch 298:  [0.3204394091550444, 0.4845860422924739]\n",
+      "Epoch 299:  [0.31568750383524585, 0.48608889263931343]\n",
+      "Epoch 300:  [0.2956457355782629, 0.48849089405017004]\n"
     ]
    }
   ],
@@ -84,19 +123,27 @@
    "batch_size = 1000\n",
    "\n",
    "\n",
-    "for eph in range(60,63): \n",
-    "    gen_checkpoint = torch.load('/eos/user/e/eneren/experiments/' + exp + \"_generator_\"+ str(eph) + \".pt\", map_location=torch.device('cpu'))\n",
+    "for eph in range(251,301): \n",
+    "    gen_checkpoint = torch.load('/eos/user/e/eneren/experiments/' + exp + \"_generator_\"+ str(eph) + \".pt\", map_location=torch.device('cuda'))\n",
    "    mGen.load_state_dict(gen_checkpoint['model_state_dict'])\n",
    "    mGen.eval()\n",
-    "    Tensor = torch.FloatTensor \n",
+    "    Tensor = torch.cuda.FloatTensor \n",
    "    z = Variable(Tensor(np.random.uniform(-1, 1, (batch_size, nz, 1, 1, 1))))\n",
-    "    elabel = torch.from_numpy(np.random.uniform(40, 40, (batch_size,1,1,1,1))).float()\n",
-    "    fake_dataG = mGen(z,elabel).detach()\n",
-    "    esumFake = F.getTotE(fake_dataG.numpy(), 30, 30, 30)\n",
-    "    esumReal = F.getTotE(showers, 30, 30, 30)\n",
-    "    JSD = F.jsdHist(esumReal, esumFake, 50, 0, 100, eph, debug=True)\n",
    "    \n",
-    "    print ('Epoch {}: '.format(eph), JSD)\n",
+    "    jsd = []\n",
+    "    for e in [40, 50]:\n",
+    "        \n",
+    "        enp = torch.from_numpy(np.random.uniform(e, e, (batch_size,1,1,1,1))).float()\n",
+    "        fake_data = mGen(z,enp).detach()\n",
+    "       \n",
+    "        esumFake = F.getTotE(fake_data.cpu().numpy(), 30, 30, 30)\n",
+    "        esumReal = F.getTotE(showers[str(e)], 30, 30, 30)\n",
+    "    \n",
+    "        jsd.append(F.jsdHist(esumReal, esumFake, 50, 0, 100, eph, debug=False))\n",
+    "        \n",
+    "    \n",
+    "    \n",
+    "    print ('Epoch {}: '.format(eph), jsd)\n",
    "    #F.plot_image2D(fake_dataG)\n",
    "    \n",
    "    "
@@ -117,7 +164,7 @@
   "metadata": {},
   "outputs": [],
   "source": [
-    "F.getTotE(showers, 30, 30, 30)"
+    "F.plot_image2D(fake_dataG)"
   ]
  },
  {

 %% Cell type:code id: tags:

 ``` python
 import torch
 import torch.nn as nn
 import sys
 sys.path.append('/home/jovyan/pytorchjob')
 from models.generator import DCGAN_G
 from torch.autograd import Variable
 import interactive.functions as F
 import numpy as np
 import matplotlib.pyplot as plt
 import matplotlib as mpl
 import h5py
 ```

 %% Cell type:code id: tags:

 ``` python
 ## G4
-f = h5py.File('/eos/user/e/eneren/run_prod20k_40GeVp2/pion-shower_40.hdf5', 'r')
+f40 = h5py.File('/eos/user/e/eneren/run_prod20k_40GeVp2/pion-shower_40.hdf5', 'r')
+f50 = h5py.File('/eos/user/e/eneren/scratch/50GeV75k.hdf5', 'r')
 ```

-%% Output
+%% Cell type:code id: tags:

-    ---------------------------------------------------------------------------
-    OSError                                   Traceback (most recent call last)
-    <ipython-input-3-e9542526b0da> in <module>
-          1 ## G4
-    ----> 2 f = h5py.File('/eos/user/e/eneren/run_prod20k_40GeVp2/pion-shower_40.hdf5', 'r')
-
-    /usr/local/lib/python3.6/dist-packages/h5py/_hl/files.py in __init__(self, name, mode, driver, libver, userblock_size, swmr, rdcc_nslots, rdcc_nbytes, rdcc_w0, track_order, **kwds)
-        406                 fid = make_fid(name, mode, userblock_size,
-        407                                fapl, fcpl=make_fcpl(track_order=track_order),
-    --> 408                                swmr=swmr)
-        409
-        410             if isinstance(libver, tuple):
-    /usr/local/lib/python3.6/dist-packages/h5py/_hl/files.py in make_fid(name, mode, userblock_size, fapl, fcpl, swmr)
-        171         if swmr and swmr_support:
-        172             flags |= h5f.ACC_SWMR_READ
-    --> 173         fid = h5f.open(name, flags, fapl=fapl)
-        174     elif mode == 'r+':
-        175         fid = h5f.open(name, h5f.ACC_RDWR, fapl=fapl)
-    h5py/_objects.pyx in h5py._objects.with_phil.wrapper()
-    h5py/_objects.pyx in h5py._objects.with_phil.wrapper()
-    h5py/h5f.pyx in h5py.h5f.open()
-    OSError: Unable to open file (unable to open file: name = '/eos/user/e/eneren/run_prod20k_40GeVp2/pion-shower_40.hdf5', errno = 13, error message = 'Permission denied', flags = 0, o_flags = 0)
+``` python
+showers50 = f50['ecal/layers'][:1000]
+showers40 = f40['ecal/layers'][:1000]
+```

 %% Cell type:code id: tags:

 ``` python
-showers = f['ecal/layers'][:]
+showers  = {
+    '50': showers50,
+    '40': showers40
+}
 ```

 %% Cell type:code id: tags:

 ``` python
 ngf = 32
 nz=100
 mGen = DCGAN_G(ngf, nz)
 mGen = nn.parallel.DataParallel(mGen)
 exp='wganv1'
 batch_size = 1000


-for eph in range(60,63):
-    gen_checkpoint = torch.load('/eos/user/e/eneren/experiments/' + exp + "_generator_"+ str(eph) + ".pt", map_location=torch.device('cpu'))
+for eph in range(251,301):
+    gen_checkpoint = torch.load('/eos/user/e/eneren/experiments/' + exp + "_generator_"+ str(eph) + ".pt", map_location=torch.device('cuda'))
    mGen.load_state_dict(gen_checkpoint['model_state_dict'])
    mGen.eval()
-    Tensor = torch.FloatTensor
+    Tensor = torch.cuda.FloatTensor
    z = Variable(Tensor(np.random.uniform(-1, 1, (batch_size, nz, 1, 1, 1))))
-    elabel = torch.from_numpy(np.random.uniform(40, 40, (batch_size,1,1,1,1))).float()
-    fake_dataG = mGen(z,elabel).detach()
-    esumFake = F.getTotE(fake_dataG.numpy(), 30, 30, 30)
-    esumReal = F.getTotE(showers, 30, 30, 30)
-    JSD = F.jsdHist(esumReal, esumFake, 50, 0, 100, eph, debug=True)

-    print ('Epoch {}: '.format(eph), JSD)
+    jsd = []
+    for e in [40, 50]:
+
+        enp = torch.from_numpy(np.random.uniform(e, e, (batch_size,1,1,1,1))).float()
+        fake_data = mGen(z,enp).detach()
+
+        esumFake = F.getTotE(fake_data.cpu().numpy(), 30, 30, 30)
+        esumReal = F.getTotE(showers[str(e)], 30, 30, 30)
+
+        jsd.append(F.jsdHist(esumReal, esumFake, 50, 0, 100, eph, debug=False))
+
+
+
+    print ('Epoch {}: '.format(eph), jsd)
    #F.plot_image2D(fake_dataG)


 ```

 %% Output

-    ---------------------------------------------------------------------------
-    PermissionError                           Traceback (most recent call last)
-    <ipython-input-4-4d3890daf5c7> in <module>
-          8
-          9 for eph in range(60,63):
-    ---> 10     gen_checkpoint = torch.load('/eos/user/e/eneren/experiments/' + exp + "_generator_"+ str(eph) + ".pt", map_location=torch.device('cpu'))
-         11     mGen.load_state_dict(gen_checkpoint['model_state_dict'])
-         12     mGen.eval()
-    /usr/local/lib/python3.6/dist-packages/torch/serialization.py in load(f, map_location, pickle_module, **pickle_load_args)
-        577         pickle_load_args['encoding'] = 'utf-8'
-        578
-    --> 579     with _open_file_like(f, 'rb') as opened_file:
-        580         if _is_zipfile(opened_file):
-        581             # The zipfile reader is going to advance the current file position.
-    /usr/local/lib/python3.6/dist-packages/torch/serialization.py in _open_file_like(name_or_buffer, mode)
-        228 def _open_file_like(name_or_buffer, mode):
-        229     if _is_path(name_or_buffer):
-    --> 230         return _open_file(name_or_buffer, mode)
-        231     else:
-        232         if 'w' in mode:
-    /usr/local/lib/python3.6/dist-packages/torch/serialization.py in __init__(self, name, mode)
-        209 class _open_file(_opener):
-        210     def __init__(self, name, mode):
-    --> 211         super(_open_file, self).__init__(open(name, mode))
-        212
-        213     def __exit__(self, *args):
-    PermissionError: [Errno 13] Permission denied: '/eos/user/e/eneren/experiments/wganv1_generator_60.pt'
+    Epoch 251:  [0.3804679251946886, 0.34817149733098474]
+    Epoch 252:  [0.46051294878083926, 0.338569981047804]
+    Epoch 253:  [0.49756609463285095, 0.28872701740420126]
+    Epoch 254:  [0.5294431896884374, 0.33102840750614887]
+    Epoch 255:  [0.5104405441541784, 0.32231291300401893]
+    Epoch 256:  [0.48537347466233505, 0.320526721042005]
+    Epoch 257:  [0.44430318353292403, 0.33754094942412155]
+    Epoch 258:  [0.41426411913761607, 0.3383523881572018]
+    Epoch 259:  [0.4406027102881503, 0.3343011917750107]
+    Epoch 260:  [0.396682637265716, 0.3473503311876263]
+    Epoch 261:  [0.40394391363113635, 0.36565837165899845]
+    Epoch 262:  [0.42483844096641477, 0.32469973222063503]
+    Epoch 263:  [0.4019302446826285, 0.36214573291228813]
+    Epoch 264:  [0.4126153084092838, 0.38613505292409905]
+    Epoch 265:  [0.4165755030129357, 0.3519977789394489]
+    Epoch 266:  [0.3913068204960252, 0.38645762498974795]
+    Epoch 267:  [0.39439823796062096, 0.3708618579692532]
+    Epoch 268:  [0.40663386822619313, 0.37241345442978496]
+    Epoch 269:  [0.3683487445416732, 0.41097496954033574]
+    Epoch 270:  [0.3931176707467255, 0.36655001291434963]
+    Epoch 271:  [0.36265554202525746, 0.38067712153366035]
+    Epoch 272:  [0.36238044571407557, 0.3891110817723188]
+    Epoch 273:  [0.3443907936692471, 0.41633865859471425]
+    Epoch 274:  [0.3754340116921492, 0.3977884345978468]
+    Epoch 275:  [0.3552435525173964, 0.4127042666470706]
+    Epoch 276:  [0.3043411019678306, 0.4198835584501347]
+    Epoch 277:  [0.34320331541360355, 0.3928834088206806]
+    Epoch 278:  [0.31577926421268404, 0.4058244392791663]
+    Epoch 279:  [0.3009843850678768, 0.4373018678372319]
+    Epoch 280:  [0.3273260324046915, 0.4398480157854807]
+    Epoch 281:  [0.30707030715104516, 0.4339921708642802]
+    Epoch 282:  [0.2915953909635101, 0.4176873424583225]
+    Epoch 283:  [0.2924977435537306, 0.43666767755248453]
+    Epoch 284:  [0.2919643384290543, 0.45133477022104457]
+    Epoch 285:  [0.2947872964957196, 0.449896095813128]
+    Epoch 286:  [0.2897446726869525, 0.48724052437850385]
+    Epoch 287:  [0.31118635032181113, 0.4325288883457097]
+    Epoch 288:  [0.2696458856150868, 0.4629579633785893]
+    Epoch 289:  [0.28041806669044533, 0.46011915244113094]
+    Epoch 290:  [0.3074237951720241, 0.46164573769828265]
+    Epoch 291:  [0.28446450303954507, 0.463811909399871]
+    Epoch 292:  [0.2849400618724373, 0.4441966401101374]
+    Epoch 293:  [0.2826640112838286, 0.4795805721176796]
+    Epoch 294:  [0.2825341470608504, 0.4795195996538825]
+    Epoch 295:  [0.2805225945655388, 0.45946307781001905]
+    Epoch 296:  [0.28991007237838295, 0.46572437861315613]
+    Epoch 297:  [0.2668600287706407, 0.5065609989888424]
+    Epoch 298:  [0.3204394091550444, 0.4845860422924739]
+    Epoch 299:  [0.31568750383524585, 0.48608889263931343]
+    Epoch 300:  [0.2956457355782629, 0.48849089405017004]

 %% Cell type:code id: tags:

 ``` python
 fake_dataG.shape
 ```

 %% Cell type:code id: tags:

 ``` python
-F.getTotE(showers, 30, 30, 30)
+F.plot_image2D(fake_dataG)
 ```

 %% Cell type:code id: tags:

 ``` python
 ```

--- a/interactive/functions.py
+++ b/interactive/functions.py
@@ -70,7 +70,7 @@ def jsdHist(data_real, data_fake, nbins, minE, maxE, eph, debug=False):
            range=[minE, maxE])
    pSEb = axSE.hist(data_fake, bins=nbins, 
            weights=np.ones_like(data_fake)/(float(len(data_fake))),
-            histtype='step', color='black',
+            histtype='step', color='red',
             range=[minE, maxE])

    frq1 = pSEa[0]
@@ -79,8 +79,10 @@ def jsdHist(data_real, data_fake, nbins, minE, maxE, eph, debug=False):
    JSD = dist.jensenshannon(frq1, frq2)    

    if (debug):
-        plt.savefig('./jsd/esum40_'+str(eph)+'.png')
+        plt.savefig('./jsd/esum_'+str(eph)+'.png')

+    plt.close()
+    
    if len(frq1) != len(frq2):
        print('ERROR JSD: Histogram bins are not matching!!')
    return JSD

--- a/pytorch_job_wgan_nccl.yaml
+++ b/pytorch_job_wgan_nccl.yaml
@@ -37,7 +37,7 @@ spec:
              command: [sh, -c] 
              args:  
                - cp /secret/krb-secret-vol/krb5cc_1000 /tmp/krb5cc_0 && chmod 600 /tmp/krb5cc_0 
-                  && python -u wgan.py --backend nccl --epochs 100 --exp wganv1 --chpt --chpt_eph 150 --lrGen 0.00001 --ncrit 5;
+                  && python -u wgan.py --backend nccl --epochs 50 --exp wganv1 --chpt --chpt_eph 250 --lrGen 0.00001 --ncrit 5;
              volumeMounts:
              - name: eos
                mountPath: /eos
@@ -49,7 +49,7 @@ spec:
                limits:
                  nvidia.com/gpu: 1
    Worker:
-      replicas: 2
+      replicas: 1
      restartPolicy: OnFailure
      template:
        metadata:
@@ -81,7 +81,7 @@ spec:
              command: [sh, -c] 
              args:  
                - cp /secret/krb-secret-vol/krb5cc_1000 /tmp/krb5cc_0 && chmod 600 /tmp/krb5cc_0 
-                  && python -u wgan.py --backend nccl --epochs 100 --exp wganv1 --chpt --chpt_eph 150 --lrGen 0.00001 --ncrit 5;
+                  && python -u wgan.py --backend nccl --epochs 50 --exp wganv1 --chpt --chpt_eph 250 --lrGen 0.00001 --ncrit 5;
              volumeMounts:
              - name: eos
                mountPath: /eos