Network System Laboratory - Backup diff of JupyterHub/DockerSpawner vs current(No. 10)

View the diff.
View the source.
Go to JupyterHub/DockerSpawner.

--- 10: 2021-07-12 (Mon) 10:15:51 iseki
+++ Cur: 2021-10-16 (Sat) 14:22:59 iseki
@@ Line 11: / Line 11: @@
  or
  (jupyterhub) # pip install dockerspawner
+- pip --no-cache-dir install git+https://github.com/jupyterhub/dockerspawner.git　をお勧め
+#br
 *** 設定 [#i5887782]
@@ Line 20: / Line 24: @@
 #br
-*** 起動用 Docker images [#w0091afe]
+*** Docker Images [#ta8920e4]
+**** [[jupyterhub-sigleuser>./jupyterhub-sigleuser]] [#zf5de082]
 - docker pull jupyterhub/singleuser
+#br
+**** datascience-notebook  [#s5b3175e]
 - docker pull jupyter/datascience-notebook
 #br
@@ Line 42: / Line 50: @@
   && rm -rf /var/lib/apt/lists/*
-**** 二重引数の問題が発生 [#t9a36c00]
+**** Docker commit でイメージを作ると，二重引数の問題が発生 [#t9a36c00]
 - iseki でログイン時
  # docker exec -it jupyter-iseki /bin/bash
@@ Line 64: / Line 72: @@
 cp /etc/subuid- /etc/subuid
-**** [[start.sh>./start.sh]] [#q31443f9]
+- [[解決>#xeab91b6]]
+#br
+**** [[start.sh（改）>./start.sh-kai]],  [[start.sh（オリジナル）>./start.sh]][#q31443f9]
 - コンテナ内の /usr/local/bin/start.sh
 -- コンテナ内の環境設定スクリプト
+- 利用できる環境変数
+ SHELL=/bin/bash
+ USER_ID=10015
+ MINIFORGE_VERSION=4.9.2-0
+ HOSTNAME=58f17edda0c7
+ LANGUAGE=en_US.UTF-8
+ JUPYTERHUB_API_TOKEN=d8137932ac144ebaa70bb04ba359fbac
+ JUPYTERHUB_BASE_URL=/
+ NB_UID=10015
+ PWD=/home/teacher/iseki
+ NB_NOTEDIRX=/home/{groupname}/{username}/work
+ GRANT_SUDO=yes
+ JUPYTERHUB_SERVER_NAME=
+ HOME=/home/teacher/iseki
+ LANG=en_US.UTF-8
+ JPY_API_TOKEN=d8137932ac144ebaa70bb04ba359fbac
+ NB_GID=10001
+ CHOWN_HOME=yes
+ JUPYTERHUB_SERVICE_PREFIX=/user/iseki/
+ JUPYTERHUB_OAUTH_CALLBACK_URL=/user/iseki/oauth_callback
+ NB_GROUP=teacher
+ USER=iseki
+ SHLVL=1
+ CONDA_DIR=/opt/conda
+ JUPYTERHUB_API_URL=http://172.22.1.75:8081/hub/api
+ JUPYTERHUB_CLIENT_ID=jupyterhub-user-iseki
+ JUPYTERHUB_HOST=
+ CONDA_VERSION=4.9.2
+ NB_USER=iseki
+ LC_ALL=en_US.UTF-8
+ PATH=/opt/conda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+ JUPYTERHUB_USER=iseki
+ JUPYTERHUB_ACTIVITY_URL=http://172.22.1.75:8081/hub/api/users/iseki/activity
+ DEBIAN_FRONTEND=noninteractive
+ _=/usr/bin/env
 #br
@@ Line 72: / Line 120: @@
 - /usr/local/anaconda/envs/jupyterhub/lib/python3.8/site-packages/dockerspawner/
--- __init__.py
+-- edit __init__.py
 -- ex.) nsldockerspawner.py
  ......
  from dockerspawner import DockerSpawner
  class NSLDockerSpawner(DockerSpawner):
  ......
@@ Line 87: / Line 135: @@
      .......
      return super(NSLDockerSpawner, self).start()
+#br
-*** Trouble Shooting [#j137c8ff]
+**** MDLDockerSpawner [#d9dde499]
+- 参考：  https://gist.github.com/marufeuille/596cf6f9e26fa6fcb7e304d5b163c7c8
+  # refer: https://gist.github.com/marufeuille/596cf6f9e26fa6fcb7e304d5b163c7c8
+  from dockerspawner import SystemUserSpawner
+  from traitlets import Unicode
+  import pwd, os, grp
+
+  class ProjectDockerSpawner(SystemUserSpawner):
+
+      dobleArguments = False
+      #dobleArguments = True
+
+
+      host_homedir_format_string = Unicode(
+          "/home/{groupname}/{username}",
+          config = True,
+      )
+
+      image_homedir_format_string = Unicode(
+          "/home/{groupname}/{username}",
+          config = True,
+      )
+
+
+      @property
+      def host_homedir(self):
+          if (
+              self.host_homedir_format_string is not None
+              and self.host_homedir_format_string != ""
+          ):
+              homedir = self.host_homedir_format_string.format(username=self.user.name, groupname=self.get_groupname())
+          else:
+              import pwd
+
+              homedir = pwd.getpwnam(self.user.name).pw_dir
+          return homedir
+
+
+      @property
+      def homedir(self):
+          return self.image_homedir_format_string.format(username=self.user.name, groupname=self.get_groupname())
+
+
+      def get_args(self):
+          args = super(ProjectDockerSpawner, self).get_args()
+          if (not self._user_set_cmd) and self.dobleArguments:
+              args = []
+          return args
+
+
+      #def get_command(self):
+      #    cmd = super(ProjectDockerSpawner, self).get_command()
+      #    return cmd
+      #
+      #    '''
+      #    if self._user_set_cmd:
+      #        cmd = self.cmd
+      #    else:
+      #        image_info = yield self.docker("inspect_image", self.image)
+      #        cmd = image_info["Config"]["Cmd"]
+      #    return cmd + self.get_args()
+      #    '''
+
+
+      def template_namespace(self):
+          d = super(ProjectDockerSpawner, self).template_namespace()
+          if self.group_id >= 0:
+              d['groupname'] = self.get_groupname()
+          return d
+
+
+      def get_groupname(self):
+          gname = ''
+          if self.group_id >= 0:
+              gname = grp.getgrgid(self.group_id).gr_name
+          return gname
+
+
+      def get_env(self):
+          env = super(ProjectDockerSpawner, self).get_env()
+          if self.group_id >= 0:
+              #env.update(NB_GID=self.group_id)
+              import grp
+              gname = self.get_groupname()
+              env.update(NB_GROUP=gname)
+          return env
+
+
+      def start(self):
+          name = self.user.name
+          user_data = pwd.getpwnam(name)
+          user_gid = user_data.pw_gid
+          gid_list = os.getgrouplist(name, user_gid)
+
+          work_dir = self.notebook_dir
+          self.volumes['jupyterhub-user-{username}'] = work_dir
+
+          for gid in gid_list:
+              gname = grp.getgrgid(gid).gr_name
+              if gname.startswith("project-"):
+                  dirname = gname.replace("project-", "")
+                  self.volumes[gname] = work_dir + '/projects/' + dirname
+
+          return super(ProjectDockerSpawner, self).start()
+#br
-**** 何故かイメージを変更して commit すると引数が２重になる？ （''非常に重要な問題''）[#xeab91b6]
-- docker logs のでエラーメッセージ
--- [C 2021-07-11 13:52:21.786 SingleUserNotebookApp application:89] Bad config encountered during initialization: Error loading argument SingleUserNotebookApp.ip=['0.0.0.0', '0.0.0.0'], ip only accepts one value, got 2: ['0.0.0.0', '0.0.0.0']
-- /usr/local/anaconda/envs/jupyterhub/lib/python3.8/site-packages/dockerspawner/dockerspawner.py L804
--- get_command() の cmd + self.get_args() で cmd に既に引数が入っている？
-- 最新版 (Project用) [[jupyterhub_config.py>../SystemUserSpawner/jupyterhub_config.py]] 内で対症療法中
-- ''上記の Dockerfile を使って作成すれば問題無し''
 #br
+*** Tools [#ke66d2d2]
+**** アイドルしている Docker コンポーネントを殺す！ [#uf1854cf]
+***** %%jupyterhub-idle-culler%% [#eb3404d4]
+- jupyterhub_idle_culler
+- https://github.com/jupyterhub/jupyterhub-idle-culler
+- pip で入れると No module named が出て，動かない模様
+-- No module named jupyterhub_idle_culler
+- GitHub から直接入れる．
+-- git clone https://github.com/jupyterhub/jupyterhub-idle-culler.git
+-- cd jupyterhub-idle-culler
+-- python setup.py build
+-- python setup.py build
+- ''期待通りに動かん！''
+- jupyterhub_config.py
+ lmport sys
+ c.JupyterHub.services = [
+    {
+        'name': 'idle-culler',
+        'admin': True,
+        'command': [
+            sys.executable,
+            '-m', 'jupyterhub_idle_culler',
+            '--timeout=3600'
+        ],
+    }
+ ]
+***** cull_idle_servers.py [#zc9428f7]
+- jupyterhub-idle-culler は期待通りに動かない．cull_idle_servers.py の方が良いみたい．
+- https://github.com/jupyterhub/jupyterhub/tree/a6b7e303df03865d6420f6bccdf627b39f1d0dc1/examples/cull-idle
+- 実行状態(run)も確実に殺してくれる（stop でなくて, rm）．
+- Webで開いたままだとダメ
+- jupyterhub_config.py
+ import sys
+ c.JupyterHub.services = [
+    {
+        'name': 'idle-culler',
+        'admin': True,
+        'command': [
+            sys.executable,
+            '/usr/local/bin/cull_idle_servers.py',
+            '--timeout=3600'
+        ],
+    }
+ ]
+#br
+** Trouble Shooting [#j137c8ff]
 **** 新しいファイルがパーミッションが無いとして，作成できない． [#z72fb1be]
@@ Line 108: / Line 314: @@
 #br
-** Hack [#gdd93e11]
+* Hack [#gdd93e11]
+*** カスタマイズ [#p63e4677]
+- https://jupyterhub.readthedocs.io/en/stable/reference/spawners.html
+#br
-*** Groups [#wefff3af]
+*** DockerSpawner が self.get_env() で取れる情報 [#bb38d813]
+ {
+     'JUPYTERHUB_API_TOKEN': 'cfd4fbcd6ea6491492016765dbbf4805',
+     'JPY_API_TOKEN': 'cfd4fbcd6ea6491492016765dbbf4805',
+     'JUPYTERHUB_CLIENT_ID': 'jupyterhub-user-bob',
+     'JUPYTERHUB_HOST': '',
+     'JUPYTERHUB_OAUTH_CALLBACK_URL': '/user/bob/oauth_callback',
+     'JUPYTERHUB_USER': 'bob',
+     'JUPYTERHUB_SERVER_NAME': '',
+     'JUPYTERHUB_API_URL': 'http://172.22.1.75:8081/hub/api',
+     'JUPYTERHUB_ACTIVITY_URL': 'http://172.22.1.75:8081/hub/api/users/bob/activity',
+     'JUPYTERHUB_BASE_URL': '/',
+     'JUPYTERHUB_SERVICE_PREFIX': '/user/bob/',
+     'GRANT_SUDO': 'yes',
+     'USER': 'bob',
+     'NB_USER': 'bob',
+     'USER_ID': 1001,
+     'NB_UID': 1001,
+     'HOME': '/home/users/bob',
+     'NB_GID': 100
+ }
+*** 認証モジュール [#xecc82c3]
+- /usr/local/anaconda/envs/jupyterhub/lib/python3.8/site-packages/ldapauthenticator/
+- /usr/local/anaconda/envs/jupyterhub/lib/python3.8/site-packages/ltiauthenticator/
+- /usr/local/anaconda/envs/jupyterhub/lib/python3.8/site-packages/jupyterhub/auth.py
+#br
+*** Groups毎にマウントするボリュームを変える [#wefff3af]
 - https://qiita.com/marufeuille/items/62e3a842f7a039c35aac
 #br
-*** Users [#m3f602dc]
+** Users [#m3f602dc]
 - 以下でやろうとしたこと（ユーザを jovyan以外する）は [[''dockerspawner.SystemUserSpawner''>../SystemUserSpawner]] を使用すれば解決するので，没．
 #br
@@ Line 135: / Line 374: @@
 **** start.sh [#lb268269]
-- [[start.sh>./start.sh]]
+- [[start.sh（改）>./start.sh-kai]]
 #br
+**** 起動コマンド [#mde3032e]
+ tini -g -- \
+ start-notebook.sh --ip=0.0.0.0 --port=8888 --notebook-dir=/home/teacher/iseki/jupyter --SingleUserNotebookApp.default_url=/lab
+ sudo -E -H -u iseki PATH=/opt/conda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin XDG_CACHE_HOME=/home/teacher/iseki/.cache PYTHONPATH=  \
+ jupyterhub-singleuser --ip=0.0.0.0 --port=8888 --notebook-dir=/home/teacher/iseki/jupyter --SingleUserNotebookApp.default_url=/lab
 *** Docker の起動 [#o46a3f42]
@@ Line 144: / Line 390: @@
 #br
 *** 設定ファイルから，パラメータ（引数）を渡す [#y62d4f07]
+**** 下記機能の実現は環境変数によっても可能 [#da119405]
 - file
  /usr/local/anaconda/envs/jupyterhub/lib/python3.8/site-packages/jupyterhub/spawner.py
@@ Line 149: / Line 396: @@
 -- 任意のパラメータ（引数）に ''{username}'' 等を使用可能にする
  c.DockerSpawner.args = ['--username={username}']
--- self.format_string() を通すと {...} を処理してくれる．
+-- ''self.format_string''() を通すと {...} を処理してくれる．
 - ''注：''[[SystemUserSpawner>../SystemUserSpawner]] が正しく動けば, コンテナ内の環境変数 NB_USER で取れるので，''以下のコード修正は不要''．
@@ Line 155: / Line 402: @@
 #br
-**** code [#td63949c]
+***** code [#td63949c]
-***** /usr/local/anaconda/envs/jupyterhub/lib/python3.8/site-packages/jupyterhub/spawner.py [#ba454e16]
+- /usr/local/anaconda/envs/jupyterhub/lib/python3.8/site-packages/jupyterhub/spawner.py
+- 継承を使う方がスマート
 - 1.0.0
          if self.debug:
@@ Line 182: / Line 431: @@
  +            args.append(self.format_string(arg))
           return args
       def run_pre_spawn_hook(self):