container.py 114 KB
Newer Older
Pascal Meunier's avatar
Pascal Meunier committed
1 2 3
# @package      hubzero-mw2-common
# @file         container.py
# @author       Pascal Meunier <pmeunier@purdue.edu>
4
# @copyright    Copyright (c) 2016-2018 HUBzero Foundation, LLC.
Pascal Meunier's avatar
Pascal Meunier committed
5 6
# @license      http://opensource.org/licenses/MIT MIT
#
7 8
# OpenVZ code based on previous work by Richard L. Kennell and Nicholas Kisseberth
# Docker container support for HUBzero middleware
Pascal Meunier's avatar
Pascal Meunier committed
9
#
10
# Copyright (c) 2016-2018 HUBzero Foundation, LLC.
Pascal Meunier's avatar
Pascal Meunier committed
11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#
# HUBzero is a registered trademark of HUBzero Foundation, LLC.
#

"""
container.py: class to create, manipulate, and stop containers, and processes within
Designed for simfs support by OpenVZ
Used on execution hosts by the middleware service.  This class should encapsulate OpenVZ idiosyncrasies
and in theory could be replaced by another container.py file to handle other virtualization mechanisms
Some OpenVZ synonyms:
veid = containerID = ctid 
The display number in the database is the ctid
"""
import subprocess
import socket
import os
import stat
import sys
import grp
import time
import threading

from errors import MaxwellError
52 53
from constants import CONTAINER_K, VERBOSE, RUN_DIR, SERVICE_LOG
from log import log, log_exc, setup_log
54
from user_account import make_User_account, User_account, User_account_JSON, User_account_anonymous
55
DEBUG = True
Pascal Meunier's avatar
Pascal Meunier committed
56

57 58 59 60
def make_Container(disp, machine_number, overrides={}):
  """Return a Container class or subclass instance depending on the OS and configuration"""
  try:
    if overrides["class"] == "ContainerVZ7":
61
      # print "creating OpenVZ container %d" % disp
62
      return ContainerVZ7(disp, machine_number, overrides)
63 64 65
    elif overrides["class"] == "ContainerDocker":
      # print "creating Docker container"
      return ContainerDocker(disp, machine_number, overrides)
66 67
    elif overrides["class"] == "ContainerAWS":
      return ContainerAWS(disp, machine_number, overrides)
68
    else:
69
      # print "creating default OpenVZ container"
70 71 72 73 74 75 76 77 78 79 80 81 82 83 84
      return Container(disp, machine_number, overrides)
  except KeyError:
    pass
  # try to open /etc/redhat-release, if it contains "Virtuozzo Linux release 7" use the ContainerVZ7 class
  # else return an old container
  try:
    with open("/etc/redhat-release") as file:
      release = file.read()
      if release.find("Virtuozzo Linux release 7") == 0:
        return ContainerVZ7(disp, machine_number, overrides)
  except IOError:
    # e.g., file does not exist
    pass
  return Container(disp, machine_number, overrides)
  
Pascal Meunier's avatar
Pascal Meunier committed
85 86 87 88 89
def log_subprocess(p, info=None):
  """Communicate after Popen, and log any output"""
  (stdout, stderr) = p.communicate(info)
  if len(stderr) > 0:
    if p.returncode == 0:
90
      log("success:" + stderr.rstrip())
Pascal Meunier's avatar
Pascal Meunier committed
91
    else:
92
      log("stderr: " + stderr.rstrip())
Pascal Meunier's avatar
Pascal Meunier committed
93
  if len(stdout) > 0:
94
    log("stdout: " + stdout.rstrip())
Pascal Meunier's avatar
Pascal Meunier committed
95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121

class Container():
  """A container: Virtual Private Server (VPS) uniquely identified by a veid.
  Note that new documentation uses "CTID" instead of veid: ConTainer's IDentifer (CTID)
  According to the OpenVZ Users Guide, CTIDs 0-100 are reserved and should not be used.
  OpenVZ only currently uses CTID 0 but recommends reserving CTID's 0-100 for non-use

  Paths used by OpenVZ  (see OpenVZ User's Guide Version 2.7.0-8 by SWsoft):

  ct_private  (e.g., "/vz/private/veid")
  This is a path to the Container private area where Virtuozzo Containers 4.0 keeps its private data.

  ct_root (e.g., "/vz/root/veid")
  This is a path to the Container root folder where the Container private area is mounted.

  vz_root
  This is a path to the Virtuozzo folder where Virtuozzo Containers program files are located.

  depends on script INTERNAL_PATH +"mergeauth"

"""

  def __init__(self, disp, machine_number, overrides={}):
    self.k = CONTAINER_K
    self.k.update(overrides)
    self.disp = disp
    self.vncpass = None
122
    self.mounts = []
123
    # self.ionhelper: support for instanton cache user to run the actual 
124
    # simulations to provide results to both cache and real user.
125 126 127 128
    if 'IONHELPER' in self.k:
      self.ionhelper = True
    else:
      self.ionhelper = False
Pascal Meunier's avatar
Pascal Meunier committed
129
    if disp < 1:
130
      # container ID 0 is the execution host itself so can't be used 
Pascal Meunier's avatar
Pascal Meunier committed
131 132 133
      raise MaxwellError("Container ID must be at least 1")
    if disp > int(self.k["MAX_CONTAINERS"]) or disp < 1:
      raise MaxwellError("Container ID must be less than %d" % int(self.k["MAX_CONTAINERS"]))
134
      
Pascal Meunier's avatar
Pascal Meunier committed
135 136 137
    self.veid = disp + self.k["VZOFFSET"]
    self.vz_private_path = "%s/private/%d" % (self.k["VZ_PATH"], self.veid)
    self.vz_root_path = "%s/root/%d" % (self.k["VZ_PATH"], self.veid)
138 139 140 141
    
    # IPv4 address of the container = f(PRIVATE_NET, containerID, machine_number)
    # where machine_number defaults to something based on the IP address of the execution host
    # so that containers have different IP addresses across a fleet of execution hosts
Pascal Meunier's avatar
Pascal Meunier committed
142
    #
143
    # To avoid the Invalid MIT-MAGIC-COOKIE error, the IP address of containers must be unique
Pascal Meunier's avatar
Pascal Meunier committed
144 145 146 147
    # over an entire hub (not just over an execution host).
    # xauth list shows the cookies a user has.  A cookie in use can get overwritten by a new
    # one if there's an IP address collision
    #
148 149 150 151 152 153 154 155 156 157 158 159
    # PRIVATE_NET IPv4 has (at least) 2 bytes free.  If a fleet has no more than 64 execution hosts
    # the address space for each is 256/64 * 256 = 1024 -2 (for broadcast and "subnet")
    # so there's a maximum of 1022 containers/execution host
    # 
    # automatic "machine_number" assignment
    # assumption: IPv4 is used, and the last byte of the IP address of execution hosts mod 64 is unique
    # (not necessarily true, one could be x.y.z.1 and another x.y.z.65)
    # if there's a collision, you'll need to manually set "machine_number" for one of them
    # or, set "machine_number" for all your execution hosts
    #
    if disp > 1022:
      # count starts at 1; 1023th container could be broadcast address
Pascal Meunier's avatar
Pascal Meunier committed
160 161 162
      log("container ID %d too high, xauth collisions possible" % disp)
    if (machine_number == 0):
      try:
163
        self.machine_number = int(socket.gethostbyname(socket.getfqdn()).split('.')[3])
Pascal Meunier's avatar
Pascal Meunier committed
164 165
      except StandardError:
        raise MaxwellError("machine_number not set and unable to derive one from IP address.")
166 167 168 169
      if (self.machine_number == 0):
        raise MaxwellError("unable to set machine_number")
    else:
      self.machine_number = machine_number
170
    if DEBUG:
171 172 173 174 175 176
      try:
        # virtual SSH doesn't setup a log file but uses this class to get the IP address
        # consider setting up a log file in /usr/bin/virtualssh_client
        log("DEBUG: machine number is %d" % machine_number)
      except AttributeError:
        pass
Pascal Meunier's avatar
Pascal Meunier committed
177

178 179 180
    # last byte = disp mod 256
    # second byte = int(disp/256) + (machine_number mod 64) *4
    # broadcast address if machine_number mod 64 = 63, disp/255 = 3 and last byte = 255
181 182 183
    digit = disp/256 + (self.machine_number % 64) *4
    self.veaddr = self.k["PRIVATE_NET"] % (digit % 256, disp % 256)    
    
184
  def groups(self):
185 186 187
    """ find last user in /etc/passwd, which is the user owning the tool session, and get the groups of that account.  
    HELPER accounts like "ionhelper" (see also self.k['HELPER']) have their /etc/passwd entries before the user's."""
    cmd = ["tail", "-n", '1', "/var/lib/vz/root/%s/etc/passwd" % veid]
188 189 190 191
    userline = subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate()[0]
    userparts = userline.split(':')
    return make_User_account(userparts[0], self.k).groups()

Pascal Meunier's avatar
Pascal Meunier committed
192 193 194 195 196 197 198 199 200 201
  def __printvzstats(self, err):
    """Print statistics for an OpenVZ VPS."""
    f = open("/proc/vz/vestat")
    for line in f:
      arr = line.split()
      if len(arr) < 5:
        continue
      if arr[0] == "VEID":
        continue
      if DEBUG:
202
        log("DEBUG: Checking /proc/vz/vestat veid = %s" % arr[0])
Pascal Meunier's avatar
Pascal Meunier committed
203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229
      try:
        if int(arr[0]) == self.veid:
          # Since VEs are pre-created, this is NOT the real time.
          # Let the middleware host compute this time.
          #log("real %f\n" % (int(arr[4])/1000.0))
          err.write("user %f\n" % (int(arr[1])/1000.0))
          err.write("sys %f\n" % (int(arr[3])/1000.0))
          break
      except ValueError:
        log("can't convert to integer, continuing")
    f.close()
    flag_print = False
    for line in open("/proc/user_beancounters"):
      if line.find(":") > 0:
        parts = line.split(":")
        if parts[0].strip() == "Version":
          continue
        if flag_print:
          break
        if int(parts[0].strip()) == self.disp:
          flag_print = True
          err.write("resource                     held              maxheld              barrier                limit              failcnt\n")
          err.write(parts[1].strip() + "\n")
      else:
        if flag_print:
          err.write(line.strip() + "\n")

230
  def __etc_passwd(self, account, err):
231 232
    """Add user info to /etc/passwd, /etc/shadow and /etc/group, also modify /etc/sudoers.
    """
Pascal Meunier's avatar
Pascal Meunier committed
233 234 235 236
    etc_passwd = open(self.vz_root_path + "/etc/passwd", "a")
    etc_shadow = open(self.vz_root_path + "/etc/shadow", "a")
    etc_group = open(self.vz_root_path + "/etc/group", "a")

237
    if 'apps' in account.groups():
Pascal Meunier's avatar
Pascal Meunier committed
238
      # add the apps user and edit the sudoers file to allow su to apps
239 240 241 242 243 244 245
      apps_user = make_User_account('apps', self.k)
      err.write(apps_user.passwd_entry() + "\n")
      etc_passwd.write(apps_user.passwd_entry() + "\n")
      etc_shadow.write(apps_user.shadow_entry() + "\n")
      etc_sudoers = open(self.vz_root_path + "/etc/sudoers", 'a', 0440)
      etc_sudoers.write("%apps           ALL=NOPASSWD:/bin/su - apps\n")
      etc_sudoers.close()
Pascal Meunier's avatar
Pascal Meunier committed
246 247 248 249
      apps_groups = apps_user.groups()
    else:
      apps_groups = []

250 251 252 253 254 255 256 257 258 259 260 261
    # ionhelper user for instant-on cache management
    # assumes /var/ion/... exists in the template
    if self.ionhelper:
      err.write('ionhelper:x:199:199::/var/ion/:/bin/false\n')
      etc_passwd.write('ionhelper:x:199:199::/var/ion/:/bin/false\n')
      etc_shadow.write('ionhelper:*:17821:0:99999:7:::\n')
    elif 'HELPER' in self.k:
      helper_pwd = '%s:x:%s:%s::%s:/bin/false\n' % (self.k['HELPER'], self.k['HELPER_UID'], self.k['HELPER_GID'], self.k['HELPER_HOME'])
      err.write(helper_pwd)
      etc_passwd.write(helper_pwd)
      etc_shadow.write('%s:*:17821:0:99999:7:::\n' % self.k['HELPER'])

Pascal Meunier's avatar
Pascal Meunier committed
262 263 264 265
    # write the /etc/passwd entry of the user last so we can look that up in firewall_readd.py
    err.write(account.passwd_entry())
    err.write("\n")
    etc_passwd.write(account.passwd_entry())
266
    etc_passwd.write("\n")
Pascal Meunier's avatar
Pascal Meunier committed
267
    etc_shadow.write(account.shadow_entry())
268
    etc_shadow.write("\n")
Pascal Meunier's avatar
Pascal Meunier committed
269 270 271
    # Add CMS groups to the /etc/group file in the container/VEID...
    # normally CMS gids are above 500 
    # gids below 500 could be system groups and should be left alone, except for fuse.  See later
272 273 274 275 276 277 278
    for g in account.group_pairs():
      # expecting to loop over something like this:  [['group10', 10], ['group11', 11]]
      gname = g[0]
      gid = g[1]
      if gid > 500:
        # copy all group info as is if gid > 500
        if gname in apps_groups:
279 280 281 282 283
          #  support su to apps.  Create all the groups that user apps belongs to, and add user apps and user helper if applicable
          if 'HELPER' in self.k:
            etc_group.write("%s:x:%d:%s,%s,%s\n" % (gname, gid, account.user, 'apps', self.k['HELPER']))
          else:
            etc_group.write("%s:x:%d:%s,%s,%s\n" % (gname, gid, account.user, 'apps', 'ionhelper'))
284
        else:
285 286 287 288
          if 'HELPER' in self.k:
            etc_group.write("%s:x:%d:%s,%s\n" % (gname, gid, account.user, self.k['HELPER']))
          else:
            etc_group.write("%s:x:%d:%s,%s\n" % (gname, gid, account.user, 'ionhelper'))
289 290 291 292

    if self.ionhelper:
      # add ionhelper group and put user in it
      etc_group.write("%s:x:%d:%s\n" % ('ionhelper', 199, account.user))
293 294 295
    if 'HELPER' in self.k:
      # add HELPER group and put user in it
      etc_group.write("%s:x:%s:%s\n" % (self.k['HELPER'], self.k['HELPER_GID'], account.user))
296 297 298
    etc_group.close()
    etc_passwd.close()
    etc_shadow.close()
Pascal Meunier's avatar
Pascal Meunier committed
299 300

  def update_resources(self, session):
301 302 303 304
    """
    Add contents to resources file for anonymous sessions
    Experimental anonymous session support.  Use at your own risk
    """
Pascal Meunier's avatar
Pascal Meunier committed
305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345
    homedir = self.k["HOME_DIR"]+"/"+'anonymous'
    rpath = "%s/%s/data/sessions/%s/resources" % (self.vz_root_path, homedir, session)
    try:
      rfile = open(rpath,"a+")
      # Read data from command line and write to file, until an empty string is found
      while 1:
        line = sys.stdin.readline()
        if line == "":
          break
        rfile.write(line)

      rfile.close()
    except OSError:
      raise MaxwellError("Unable to append to resource file.")

  def create_anonymous(self, session, params):
    """Add an "anonymous" user in the container"""
    homedir = self.k["HOME_DIR"]+"/"+'anonymous'
    args = ['/usr/sbin/vzctl', 'exec2', str(self.veid), 'adduser', '--uid', '1234']
    args += ['--disabled-password', '--home', homedir, '--gecos', '"anonymous user"', 'anonymous']
    subprocess.check_call(args)

    rdir = "%s%s/data/sessions/%s" % (self.vz_root_path, homedir, session)
    if VERBOSE:
      log("creating " + rdir)
    os.makedirs(rdir)
    os.chown(rdir,  1234, 1234)
    os.chown("%s%s/data/sessions" % (self.vz_root_path, homedir),  1234, 1234)
    os.chown("%s%s/data/" % (self.vz_root_path, homedir),  1234, 1234)
    rfile = open(rdir+"/resources", "w")
    rfile.write("sessionid %s\n" % session)
    rfile.write("results_directory %s/data/results/%s\n" % (homedir, session))
    os.fchown(rfile.fileno(), 1234, 1234)
    rfile.close()
    if VERBOSE:
      log("setup anonymous session directory and resources in session %s" % (session))
    if params is not None and params != "":
      import urllib2
      params_path = rdir + "/parameters.hz"
      pfile = open(params_path, "w")
      pfile.write(urllib2.unquote(params).decode("utf8"))
346
            
347
  def mount_paths(self, mountlist):
348
    """Given a list of paths, mount them in the container.  Called by maxwell_service when receiving the command mount_paths.  
349 350 351 352 353 354 355 356 357 358 359 360 361 362
    This functionality is needed for the MyGeohub hydroshare connectivity via /srv/irods/external. Need to implement it for Docker too!"""
    for mount_pt in mountlist:
      log("mounting %s" % (mount_pt))
      # mount_pt must already exist
      if not os.path.exists(mount_pt):
        log("trying to mount '%s' but it doesn't exist" % mount_pt)
        continue
      if not os.path.exists(self.vz_root_path + mount_pt):
        args = ['/bin/mkdir', '-m', '0700', '-p', self.vz_root_path + mount_pt]
        p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        p.communicate()
        if p.returncode != 0:
          raise MaxwellError("Could not create '%s'" % (self.vz_root_path + mount_pt))
      self.__root_mount(mount_pt, 'rw,noatime')
363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384

  def __setup_helper(self, helper, helper_home, helper_uid, helper_gid, session_id, runner_home, runner_name, err):
    """
    Helper account: local trusted system account supporting sudo operations on behalf of a tool session's owner
    setup resources for helper accounts which allow users "sudo" operations to be done to a trusted account
    helper account home must be local to container for security reasons, because user can write to the drivers directory
    the session user's home directory may be on NFS, don't use root to read from it
    """
    # create session directory in helper account's home
    sessiondir_helper = "%s/%s/data/sessions/%s" % (self.vz_root_path, helper_home, session_id)
    if DEBUG:
      log("creating session directory in helper account: %s" % sessiondir_helper)
    os.makedirs(sessiondir_helper, 0750)

    # copy/edit "resources" file from user to helper account's session directory
    # rewrite the results_directory line
    rpath_user = "%s/data/sessions/%s/resources" % (runner_home, session_id)
    rpath_helper = "%s/%s/data/sessions/%s/resources" % (self.vz_root_path, helper_home, session_id)
    # run copy as user to access NFS share
    # more or less like:
    # su pmeunier -s /bin/sh -c "cat /home/nanohub/pmeunier/data/sessions/20522/resources" \
    # > /vz/root/2/var/ion/data/sessions/20522/resources
385
    
386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425
    # open file for writing, unbuffered 
    resfile = open(rpath_helper, 'w', 0)
    # rewrite the first 2 lines
    resfile.write("sessionid %s\n" % session_id)
    resfile.write("results_directory %s/runs\n" % helper_home)
    # read from original starting at line 3, as the user owning the tool session
    p = subprocess.Popen(['su', runner_name, '-s', '/bin/sh', '-c', 'tail -n +3 ' + rpath_user], stdout = resfile, stderr = err)
    p.communicate()

    drivers_dir = self.vz_root_path + helper_home + '/drivers'
    try:
      os.makedirs(drivers_dir, 0770)
    except OSError:
      # it's fine if it already exists, but set permissions
      os.chmod(drivers_dir, 0770)
    os.chmod(self.vz_root_path + '/' + helper_home, 0770)
    # finally make sure everything is owned by helper account
    os.system('chown -R %s:%s %s/%s' % (helper_uid, helper_gid, self.vz_root_path, helper_home))

  def __mergeauth_X11(self, username, err):
    """
    Add X11 information for container to helper's .Xauthority file
    Use the mergeauth script wrapper to xauth
    """
    count = 0
    # helper needs to access the display
    args = ['/usr/sbin/vzctl', 'exec2', str(self.veid), 'su', username, '-s', '/bin/sh', '-c', self.k["INTERNAL_PATH"] + 'mergeauth']
    if DEBUG:
      log("DEBUG: xauth: %s\n" % " ".join(args))
    while True:
      p = subprocess.Popen(args, stderr = err, stdout = err)
      p.communicate()
      if p.returncode == 0:
        break
      time.sleep(0.5)
      count = count+1
      if count > self.k["XAUTH_RETRIES"]:
        err.write("Unable to extract xauth cookie for %d\n" % self.veid)
        raise MaxwellError() # cleanup rules

Pascal Meunier's avatar
Pascal Meunier committed
426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442
  def __child_unix_command(self, user, session_id, timeout, command, params, sesslog_path):
    """child
      1. Setup the environment inside the container: permissions, password, group files,
      2. Setup the firewall rules on the host
      3. Setup X server authentication.  Call xauth so we're allowed to connect to the X server
      4. Invoke the command within the container
      5. Calculate time stats
      6. Restore the firewall rules
    """
    err = open(sesslog_path + ".err", 'a', 0)
    out = open(sesslog_path + ".out", 'a', 0)
    err.write("Starting command '%s' for '%s' with timeout '%s'\n" % (command, user, timeout))

    if user == "anonymous":
      if self.k["ANONYMOUS"]:
        # do not mount /home.
        # do not make any LDAP calls.
443
        account = User_account_anonymous(user)
Pascal Meunier's avatar
Pascal Meunier committed
444 445 446
      else:
        raise MaxwellError("Anonymous user session not supported")
    else:
447
      account = make_User_account(user, self.k)
448 449 450
      # mount script doesn't mount /home anymore
      self.__root_mount(account.homedir, "rw,nodev,nosuid,noatime")
      # Setup the environment inside the container: permissions, password, group files
451 452 453 454 455 456
      self.__etc_passwd(account, err)
      if VERBOSE:
        err.write("VERBOSE: wrote /etc/passwd\n")
        
      if self.ionhelper:
        # ionhelper in /var/ion
457 458 459 460
        # __setup_helper(helper, helper_home, helper_uid, helper_gid, session_id, runner_home, runner_name)
        self.__setup_helper('ion', '/var/ion', '199', '199', session_id, account.homedir, user, err)
      if 'HELPER' in self.k:
        self.__setup_helper(self.k['HELPER'], self.k['HELPER_HOME'], self.k['HELPER_UID'], self.k['HELPER_GID'], session_id, account.homedir, user, err)
Pascal Meunier's avatar
Pascal Meunier committed
461

462 463 464
    # Get a list of the supplementary groups...
    groups = account.groups()
    
Pascal Meunier's avatar
Pascal Meunier committed
465 466 467
    # groups like "fuse" may pre-exist inside containers with a different gid than on the hub
    # just appending a new line to /etc/group could create conflicting definitions
    # example: vhub.org.def:@define MW_CONTAINER_GROUPS '"fuse", "public"'
468
    # If  called with two non-option arguments, adduser will add an existing user to an existing group.
Pascal Meunier's avatar
Pascal Meunier committed
469
    # Assumes a Debian Linux container, will fail with RedHat because the command is named useradd instead
470 471
    if DEBUG:
      err.write("DEBUG: calling adduser\n")
Pascal Meunier's avatar
Pascal Meunier committed
472 473 474
    for defgroup in self.k["DEFAULT_GROUPS"]:
      p = subprocess.Popen(['/usr/sbin/vzctl', 'exec2', str(self.veid), 'adduser', user, defgroup], stdout = err, stderr = err)
      p.communicate()
475 476 477
      if self.ionhelper:
        p = subprocess.Popen(['/usr/sbin/vzctl', 'exec2', str(self.veid), 'adduser', 'ionhelper', defgroup], stdout = err, stderr = err)
        p.communicate()
478 479 480
      if 'HELPER' in self.k:
        p = subprocess.Popen(['/usr/sbin/vzctl', 'exec2', str(self.veid), 'adduser', self.k['HELPER'], defgroup], stdout = err, stderr = err)
        p.communicate()
Pascal Meunier's avatar
Pascal Meunier committed
481 482 483

    # 1b. /apps bind mount conditional on apps group membership
    # This requires xvnc to not be in /apps, but in /usr/sbin
484 485
    if 'apps' in groups:
      mount_opt = 'rw,acl,noatime'
Pascal Meunier's avatar
Pascal Meunier committed
486
      self.__root_mount("/apps", mount_opt)
487 488 489 490 491
      if user == 'apps':
        err.write("user is apps, not mounting home directory twice\n")
      else:
        err.write("%s is in apps group\n" % user)
        # mount home directory of user 'apps' to support su to apps
492
        apps_account = make_User_account('apps', self.k)
493
        self.__root_mount(apps_account.homedir, "rw,nodev,nosuid,noatime")
494
    else:
495 496 497 498 499 500 501 502
      if 'APPS_SUBDIR_MOUNTING' in self.k:
        if self.k["APPS_SUBDIR_MOUNTING"]:
          # Experimental, use at your own risk
          err.write("%s is NOT in apps group, figuring out which directories to mount\n" % user)
          self.__mount_subdirs_ro(err, user, "/apps", 'ro,acl,noatime')
        else:
          mount_opt = 'ro,acl,noatime'
          self.__root_mount("/apps", mount_opt)
503 504 505
      else:
        mount_opt = 'ro,acl,noatime'
        self.__root_mount("/apps", mount_opt)
506
    
507 508 509 510 511 512 513
    if 'APPS_SUBDIR_MOUNTING' in self.k:
      # Experimental, use at your own risk
      # mount /data subdirs
      if self.k["APPS_SUBDIR_MOUNTING"]:
        if os.path.exists('/data'):
          for s in os.listdir('/data'):
            self.__mount_subdirs(err, user, os.path.join('/data',s))
514
      
515 516 517
    # Username-based mounts
    # uses bind mounts from an already mounted filesystem, which has a directory for each user
    # the mounted directory is the username
Pascal Meunier's avatar
Pascal Meunier committed
518 519 520 521 522
    if self.k["USER_MOUNT"]:
      for mount_pt in self.k["USER_MOUNT_POINTS"]:
        log("mounting %s" % (mount_pt))
        # mount_pt must already exist
        if not os.path.exists(mount_pt):
523
          err.write("Mount point '%s' does not exist" % mount_pt)
524
          continue
Pascal Meunier's avatar
Pascal Meunier committed
525 526 527 528 529 530 531 532
        source_mount = mount_pt + user
        # check if source exists
        if not os.path.exists(source_mount):
          # create it as the user, not root
          args = ['/bin/su', user, '-c', "mkdir -m 0700 " + source_mount]
          p = subprocess.Popen(args, stderr = err, stdout = err)
          p.communicate()
          if p.returncode != 0:
533 534
            if VERBOSE:
              err.write("Warning: '%s' did not exist, could not create it as user '%s', so will not be mounted\n" % (source_mount, user))
535
            continue
Pascal Meunier's avatar
Pascal Meunier committed
536 537 538 539 540 541 542 543
        if not os.path.exists(self.vz_root_path + source_mount):
          args = ['/bin/mkdir', '-m', '0700', '-p', self.vz_root_path + source_mount]
          p = subprocess.Popen(args, stderr = err, stdout = err)
          p.communicate()
          if p.returncode != 0:
            raise MaxwellError("Could not create '%s'" % (self.vz_root_path + source_mount))
        self.__root_mount(source_mount, 'rw,noatime')

544
    # Mount projects based on user membership
Pascal Meunier's avatar
Pascal Meunier committed
545 546
    if self.k["PROJECT_MOUNT"]:
      for g in groups:
547 548
        if g[0:3] == "pr-":
          source_mount = self.k["PROJECT_PATH"] + g[3:]
Pascal Meunier's avatar
Pascal Meunier committed
549
          if not os.path.exists(source_mount):
550
            continue
Pascal Meunier's avatar
Pascal Meunier committed
551 552 553 554 555 556 557
          if not os.path.exists(self.vz_root_path + source_mount):
            args = ['/bin/mkdir', '-m', '0700', '-p', self.vz_root_path + source_mount]
            p = subprocess.Popen(args, stderr = err, stdout = err)
            p.communicate()
            if p.returncode != 0:
              raise MaxwellError("Could not create '%s'" % (self.vz_root_path + source_mount))
          self.__root_mount(source_mount, 'rw,noatime')
558

559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592
    # Mount public project file areas
    # https://mygeohub.org/support/ticket/1226
    # IMPORTANT:  path for this can be entirely different from path for regular project areas
    # compare "/srv/mygeohub/projects/" vs "/srv/irods/"
    # this section uses PROJECT_PUBLIC_PATH instead of PROJECT_PATH
    # mount read-only for users who do not belong to the corresponding projects
    # mount read-write for users who belong
    # public project file areas are:  /srv/mygeohub/projects/*/files/public
    # algorithm
    #   for each project group in which the user belongs
    #     if there's a public area, mount it r/w
    #   find projects that have "public" directories
    #     if already mounted, continue
    #     else mount read-only
    if self.k["PROJECT_PUBLIC_MOUNT"]:
      # mount read/write public areas for project members
      for g in groups:
        if g[0:3] == "pr-":
          source_mount = self.k["PROJECT_PUBLIC_PATH"] + g[3:] + "/files/public"
          if not os.path.exists(source_mount):
            continue
          if not os.path.exists(self.vz_root_path + source_mount):
            args = ['/bin/mkdir', '-m', '0700', '-p', self.vz_root_path + source_mount]
            p = subprocess.Popen(args, stderr = err, stdout = err)
            p.communicate()
            if p.returncode != 0:
              raise MaxwellError("Could not create '%s'" % (self.vz_root_path + source_mount))
          self.__root_mount(source_mount, 'rw,noatime')
      import glob
      listing = glob.glob(self.k["PROJECT_PUBLIC_PATH"]+ '/*/files/public')
      for pubpath in listing:
        mntpt = self.vz_root_path + pubpath
        if os.path.isdir(mntpt):
          continue
593 594 595 596 597
        args = ['/bin/mkdir', '-m', '0700', '-p', mntpt]
        p = subprocess.Popen(args, stderr = err, stdout = err)
        p.communicate()
        if p.returncode != 0:
          raise MaxwellError("Could not create '%s'" % (mntpt))
598 599
        self.__root_mount(pubpath, 'ro,noatime')
        
600
    # Use local storage for the session directory
601
    # implemented for cdmhub, to use SSDs on the execution host
602 603 604
    if self.k["LOCAL_SESSIONDIR"]:
      import shutil
      localhome_dir="/home/sessions/" + user
605 606
      local_dir = "/home/sessions/%s/%s" % (user, session_id)
      remote_dir = account.homedir + "/data/sessions/%s" % session_id
607
      # create local user home so operations can be made under the user account and not root
608 609 610 611 612 613 614
      # mv fails if the session directory is not empty
      # but this can happen only during testing because the same sessnum is used.
      # Better to not attempt rm -rf otherwise
      if False:
        args = ['/bin/rm', '-rf', localhome_dir]
        p = subprocess.Popen(args, stderr = err, stdout = err)
        p.communicate()
615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639
      args = ['mkdir', '-p', '-m', '0750', localhome_dir]
      p = subprocess.Popen(args, stderr = err, stdout = err)
      p.communicate()
      if p.returncode != 0:
        raise MaxwellError("Could not create '%s'" % (localhome_dir))
      os.chown(localhome_dir, account.uid, account.gid)

      # move the session directory to fast local storage
      args = ['/bin/su', user, '-c', 'mv %s %s' % (remote_dir, localhome_dir)]
      p = subprocess.Popen(args, stderr = err, stdout = err)
      p.communicate()
      if p.returncode != 0:
        raise MaxwellError("Could not move '%s' to '%s'" % (remote_dir, localhome_dir))

      # mount it
      # mount: only root can do that
      # mounting as root will follow symlinks so would be unsafe, can mount over important mount points like /apps
      # create a symlink from remote_dir to local_dir, as the user
      # so the openvz mount script has to mount /home/sessions inside the container!
      args = ['/bin/su', user, '-c', "ln -s %s %s" % (local_dir, remote_dir) ]
      p = subprocess.Popen(args, stderr = err, stdout = err)
      p.communicate()
      if p.returncode != 0:
        raise MaxwellError("Could not symlink %s to '%s'" % (remote_dir, local_dir))

Pascal Meunier's avatar
Pascal Meunier committed
640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667
    # Deprecated -- SSHFS-based user mounts -- Deprecated
    if self.k["SSHFS_MOUNT"]:
      # create an SSH connection for each container
      for mount_pair in self.k["SSHFS_MOUNT_POINTS"]:
        # array of remote, local info
        remote_path = mount_pair[0] + user
        container_path = self.vz_root_path + mount_pair[1] + user
        manage_path = mount_pair[1] + user
        log("mounting %s at %s" % (remote_path, container_path))
        if not os.path.exists(manage_path):
          # create it as the user, not root
          args = ['/bin/su', user, '-c', "mkdir -m 0700 " + manage_path]
          p = subprocess.Popen(args, stderr = err, stdout = err)
          p.communicate()
          if p.returncode != 0:
            # possible race condition if user starts two sessions quickly for the first time
            raise MaxwellError("Could not create '%s'" % (manage_path))
        if not os.path.exists(container_path):
          args = ['/bin/mkdir', '-m', '0700', '-p', container_path]
          p = subprocess.Popen(args, stderr = err, stdout = err)
          p.communicate()
          if p.returncode != 0:
            raise MaxwellError("Could not create '%s'" % (self.vz_root_path + source_mount))
        args = ['/usr/bin/ssh', '-o', 'intr', '-o', 'sync_read', '-o', 'IdentityFile=%s' % self.k["SSHFS_MOUNT_KEY"], '-o', 'allow_other', remote_path, container_path]
        p = subprocess.Popen(args, stderr = err, stdout = err)
        p.communicate()

    # 2. Setup the firewall rules on the host
668 669
    if VERBOSE:
      err.write("VERBOSE: setting up firewall rules\n")
Pascal Meunier's avatar
Pascal Meunier committed
670 671 672 673 674
    self.firewall_by_group(groups, 'add')

    # Wrap the following in a "try" to reverse the iptables state in case of an exception
    try:
      # 3. Setup X server authentication
675
      self.__mergeauth_X11(user, err)
676
      if self.ionhelper:
677 678 679
        self.__mergeauth_X11('ionhelper', err)
      if 'HELPER' in self.k:
        self.__mergeauth_X11(self.k['HELPER'], err)
680

Pascal Meunier's avatar
Pascal Meunier committed
681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701
      # 4. Invoke the command within the container
      # Actual application start!
      # In the command below, use "time" to get the runtime of the command.
      # The user and sys cputimes will be inaccurate, but will be overridden
      # by printvzstats().  We use it to get the clock time of the command.
      #
      # Note: time is a built-in shell command, there is no separate binary installed
      # so the whole thing has to be passed to a shell for interpretation...
      # when shell=True, args needs to be a string for "time" to be interpreted as the built-in
      #
      # Also, the environment needs to be passed inside the container.  Setting env= in the
      # subprocess call only sets the environment for the vzctl command.
      #
      # In addition, the whole "su..." command has to be passed inside quotes, otherwise it fails.
      #  Perhaps some of the switches get interpreted by an earlier command than intended? time?
      env_cmd = " ".join(account.env(session_id, timeout, params) + ["DISPLAY=\"%s:0.0\"" % (self.veaddr)])
      try:
        env_cmd += " " + self.k["EXTRA_ENV_CMD"]
      except KeyError:
        pass
      args = ['/usr/sbin/vzctl', 'exec2', str(self.veid), 'su', user, '-s', '/bin/dash', '-c',
702 703 704 705 706 707 708 709 710 711 712 713 714
         '\"cd; %s %s\"' % (env_cmd, command)]
      if 'HELPER' in self.k:
        # expecting command like '/apps/jupyter/r16/middleware/invoke'
        toolname = command.split('/')[2]
        if toolname in self.k['HELPER_TOOLS']:
          if DEBUG:
            log("DEBUG: toolname %s matches %s\n" % (toolname, " ".join(self.k['HELPER_TOOLS'])))
          env_cmd = " ".join(make_User_account(self.k['HELPER'], self.k).env(session_id, timeout, params) + ["DISPLAY=\"%s:0.0\"" % (self.veaddr)])
          try:
            env_cmd += " " + self.k["EXTRA_ENV_CMD"]
          except KeyError:
            pass
          args = ['/usr/sbin/vzctl', 'exec2', str(self.veid), 'su', self.k['HELPER'], '-s', '/bin/dash', '-c',
Pascal Meunier's avatar
Pascal Meunier committed
715
         '\"cd; %s %s\"' % (env_cmd, command)]
716 717
      if DEBUG:
        log("DEBUG: command is %s\n" % " ".join(args))
Pascal Meunier's avatar
Pascal Meunier committed
718 719 720 721 722
      start_time = time.time()
      # subprocess.call(args)
      # Python docs: "The data read is buffered in memory, so do not use this method if the data size is large or unlimited."
      # Problem:  If the tool is misbehaving and has GBs of output, then root processes start
      # consuming GBs of memory!  Todo: find alternative to calling subprocess.communicate while capturing output
723
      p = subprocess.Popen(args, stderr = err, stdout = out)
Pascal Meunier's avatar
Pascal Meunier committed
724 725 726 727 728
      p.communicate()
      end_time = time.time()

      # 5. Calculate time stats
      if VERBOSE:
729
        err.write("Processing stats\n")
Pascal Meunier's avatar
Pascal Meunier committed
730 731 732 733 734 735 736 737 738 739 740
      err.write("real\t%f\n" % (end_time - start_time))
      self.__printvzstats(err)
      # everything went OK
      err.write("Exit_Status: 0\n")
      err.close()
    except StandardError, exc:
      # cleanup iptable rules
      err.write("tool session failed due to exception:'%s'\n" % exc)
      err.write("Exit_Status: 2\n")
      err.close()

741
    # 6. Leave firewall cleanup to maxwell_service, to be done after the container has stopped
Pascal Meunier's avatar
Pascal Meunier committed
742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773

    os._exit(0)

  def invoke_unix_command(self, user, session_id, timeout, command, params, sesslog_path):
    """Start a tool in the container.
     Child will invoke the command.
     Parent will log the exit status.  When we'll return, other things will happen (notify).
     When we are called, the log file has been closed and we're a dissociated process.
     Stdout and stderr have been redirected to files, so we use that for logging.
    user: string
    session_id: string (int+letter)
    timeout: int
    command: string
    """
    try:
      pid = os.fork()
    except OSError, ose:
      log("unable to fork: '%s', exiting" % ose)
      sys.exit(1)

    if pid == 0:
      self.__child_unix_command(user, session_id, timeout, command, params, sesslog_path)
    # parent
    try:
      log("Waiting for %d" % pid)
      os.waitpid(pid, 0)
    except OSError:
      pass
    return 0

  def screenshot(self, user, sessionid):
    """Support display of session screenshots for app UI.  On error, do not produce an exception."""
774
    account = make_User_account(user, self.k)
Pascal Meunier's avatar
Pascal Meunier committed
775 776 777 778 779 780 781 782 783 784 785
    destination = "%s/data/sessions/%s/screenshot.png" % (account.homedir, sessionid)
    if os.path.isdir("%s/data/sessions/%s" % (account.homedir, sessionid)):
      vz_env = account.env(sessionid, 8000, False)
      env_cmd = " ".join(vz_env + ["DISPLAY=\"%s:0.0\"" % (self.veaddr)])
      command = "/usr/bin/screenshot %s" % destination
      args = ['/usr/sbin/vzctl', 'exec2', str(self.veid), 'su', user, '-s', '/bin/dash', '-c',
        '\"cd; %s %s\"' % (env_cmd, command)]
      p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
      log_subprocess(p)

  def __root_mount(self, point, perm):
786
    """Mount a directory to make it available to OpenVZ containers."""
Pascal Meunier's avatar
Pascal Meunier committed
787 788 789 790 791 792 793 794
    mntpt = self.vz_root_path + point
    if os.path.isdir(mntpt):
      try:
        os.rmdir(mntpt)
      except OSError, exc:
        log("exception:'%s'\n" % exc)
        raise MaxwellError("'%s' already exists and is probably already mounted.  Giving up."
          % mntpt)
795 796 797 798 799
    try:
      saved_umask = os.umask(0)
      os.makedirs(mntpt, 0755)
    finally:
      os.umask(saved_umask)
Pascal Meunier's avatar
Pascal Meunier committed
800 801 802 803 804 805 806 807 808 809 810 811 812 813
    if VERBOSE:
      log("Created %s" % mntpt)

    # -n: Mount without writing in /etc/mtab.
    args = ["/bin/mount", "-n", "--bind", point, '-o', perm, mntpt]
    p = subprocess.Popen(args, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
    log_subprocess(p)
    if p.returncode != 0:
      try:
        os.rmdir(mntpt)
      except OSError:
        pass
      raise MaxwellError("Could not mount %s in '%d'" % (point, self.veid))

814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864
  def __mount_subdirs(self, err, user, topdir):
    """Bind mount subdirectories of topdir if they are accessible to user.  Determine if they should be read-only or r/w"""
    if os.path.isdir(topdir):
      for o in os.listdir(topdir):
        if o == 'lost+found':
          continue
        fp=os.path.join(topdir,o)
        if os.path.isdir(fp):
          accessible = False
          # try to ls the directory, as the user
          args = ['/bin/su', user, '-c', "ls " + fp]
          with open(os.devnull, 'w') as devnull:
            returncode = subprocess.call(['/bin/su', user, '-c', "ls " + fp], stdout=devnull, stderr=devnull)
          if returncode == 0:
            accessible = True
            permissions = 'ro,noatime'
          # try to touch a file as the user, with "su"
          with open(os.devnull, 'w') as devnull:
            returncode = subprocess.call(['/bin/su', user, '-c', "touch %s/.test" % fp], stdout=devnull, stderr=devnull)
          if returncode == 0:
            # we can write
            accessible = True
            permissions = 'rw,noatime'
          if accessible:
            err.write("mounting %s with permissions %s; " %(fp, permissions))
            self.__root_mount(fp, permissions)
          else:
            err.write("cannot mount %s" %(fp))

  def __mount_subdirs_ro(self, err, user, topdir, permissions):
    """Bind mount subdirectories of topdir with provided permissions if they are readable by the user.
    The permissions are intended to contain 'ro', as it only tests read access.  Faster than __mount_subdirs"""
    if os.path.isdir(topdir):
      for o in os.listdir(topdir):
        if o == 'lost+found':
          continue
        fp=os.path.join(topdir,o)
        if os.path.isdir(fp):
          accessible = False
          # try to ls the directory, as the user
          args = ['/bin/su', user, '-c', "ls " + fp]
          with open(os.devnull, 'w') as devnull:
            returncode = subprocess.call(['/bin/su', user, '-c', "ls " + fp], stdout=devnull, stderr=devnull)
          if returncode == 0:
            accessible = True
          if accessible:
            err.write("mounting %s with permissions %s; " %(fp, permissions))
            self.__root_mount(fp, permissions)
          else:
            err.write("cannot mount %s" %(fp))

Pascal Meunier's avatar
Pascal Meunier committed
865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890
  def firewall_by_group(self, groups, operation='add'):
    """groups is an array of groups the user belongs to
    FW_GROUP_MAP is an array of ["group_name", net_cidr, portmin, portmax]
    where net_cidr is something like 128.46.19.160/32
    """
    rule_start = [self.k["FW_CHAIN"], '-i', 'venet0', '-s', self.veaddr]
    for g in self.k["FW_GROUP_MAP"]:
      if g[0] in groups:
        (net_cidr, portmin, portmax) = g[1:]
        if net_cidr == "":
          # block access to hubzero networks due to firewall rule exceptions
          hostpart = ['!', '-d', self.k["MW_PROTECTED_NETWORK"]]
        else:
          # hostpart = ['-d', socket.gethostbyname(host)]
          hostpart = ['-d', net_cidr]
        if portmin == 0:
          # all ports
          portpart = []
        else:
          if portmax == 0:
            # single port
            portpart = ['-p', 'tcp', '--dport', '%s' % portmin]
          else:
            portpart = ['-p', 'tcp', '--dport', '%s:%s' % (portmin, portmax)]
        fwd_rule = rule_start + hostpart + portpart + ['-j', 'ACCEPT']
        if DEBUG:
891
          log('DEBUG: ' + operation + ' ' + ' '.join(fwd_rule))
Pascal Meunier's avatar
Pascal Meunier committed
892
        if operation == 'add':
893 894 895 896 897 898 899 900 901 902 903 904
          i=0
          while i < 3:
            # retry adding firewall rules.  Log final failure and keep going.
            try:
              # insert due to RETURN or DROP rules
              subprocess.check_call(['/sbin/iptables', '-A'] + fwd_rule)
              i = 3
            except subprocess.CalledProcessError:
              log('Warning: unable to add firewall rule ' + ' '.join(fwd_rule))
              i += 1
              if i < 3:
                time.sleep(1)
Pascal Meunier's avatar
Pascal Meunier committed
905
        else:
906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932
          # delete rule if present
          # iptables -C uses the logic for -D (delete) so it returns 0 if the rule exists, 1 if it doesn't exist
          args = ['/sbin/iptables', '-t', 'filter', '-C'] + fwd_rule
          p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
          p.communicate()
          if p.returncode ==0:
            # retry deleting firewall rules, but eventually ignore the error
            i=0
            while i < 10:
              try:
                subprocess.check_call(['/sbin/iptables', '-D'] + fwd_rule)
                i = 10
              except subprocess.CalledProcessError:
                log('Warning: unable to delete firewall rule ' + ' '.join(fwd_rule))
                i += 1
                if i < 10:
                  time.sleep(1)

  def firewall_cleanup(self):
    """Get all rules for the AUTO_FORWARD chain.  Delete the ones that include the IP address of this container"""
    p = subprocess.Popen(['/sbin/iptables', '-S', self.k["FW_CHAIN"]], stderr=subprocess.PIPE, stdout=subprocess.PIPE)
    (stdout, stderr) = p.communicate()
    if p.returncode != 0:
      log("Unable to list rules in %s chain" % self.k["FW_CHAIN"])
      return
    lines= stdout.split('\n')
    for line in lines:
933
      if line.find(self.veaddr + '/32') != -1:
934 935 936 937 938 939 940 941 942 943 944
        args = ['/sbin/iptables', '-D'] + line.split()[1:]
        i = 0
        while i < 10:
          try:
            subprocess.check_call(args)
            i = 10
          except subprocess.CalledProcessError:
            log('Warning: unable to delete firewall rule: ' + ' '.join(args))
            i += 1
            if i < 10:
              time.sleep(1)
Pascal Meunier's avatar
Pascal Meunier committed
945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983

  def set_ipaddress(self):
    """Set IP address of container"""
    status = os.system("vzctl set %d --ipadd %s" % (self.veid, self.veaddr))
    if status != 0:
      raise MaxwellError("Bad status for vzctl set %d --ipadd %s: %d" %
        (self.veid,self.veaddr,status))

  def umount(self):
    """unmount container directories"""
    self.__openVZ_umount(self.vz_root_path)
    self.__openVZ_umount(self.vz_private_path)

  def __openVZ_umount(self, fs_path):
    """If given path exists, call ctid.umount for that container
    The ctid.umount script is part of the shutdown process of a container.  This indicates
    an unclean shutdown.
    """
    if os.path.exists(fs_path):
      # tell openVZ to unmount that container's file system
      # internally, that umount script doesn't use abolute paths so we need
      # to set the PATH
      v_env = {"VEID" : str(self.veid), "PATH": "/bin:/usr/bin"}
      args = ["%s/%s" % (self.k["VZ_CONF_PATH"], self.k["OVZ_SESSION_UMOUNT"])]
      p = subprocess.Popen(args, stderr=subprocess.PIPE, stdout=subprocess.PIPE, env = v_env)
      log_subprocess(p)
      if p.returncode != 0:
        raise MaxwellError("Could not unmount container '%d'" % (self.veid))
      if self.k["APPS_READONLY"]:
        # we mount /apps rw or ro depending on group membership of user
        try:
          os.rmdir(fs_path + '/apps')
        except OSError:
          pass
      try:
        os.rmdir(fs_path)
      except OSError:
        pass
      if os.path.exists(fs_path):
984 985 986 987 988 989 990 991
        if os.path.exists('/usr/sbin/lsof'):
          p = subprocess.Popen(['/usr/sbin/lsof', fs_path], stderr=subprocess.PIPE, stdout=subprocess.PIPE)
          (stdout, stderr) = p.communicate()
          if p.returncode == 0:
            log("lsof %s: %s" % (fs_path, stdout))
          else:
            log("Unable to call lsof on %s" % fs_path)        
        raise MaxwellError("'%s' still exists" % fs_path)
Pascal Meunier's avatar
Pascal Meunier committed
992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045

  def create_xstartup(self):
    """xstartup is a file we create to make VNC happy.  RUN_DIR is something like '/usr/lib/mw'.
    VNC is started inside containers, and /usr is mounted inside."""
    x_path = RUN_DIR + "/xstartup"
    try:
      lock_stat = os.lstat(x_path)
    except OSError:
      # does not exist
      try:
        xstartup = os.open(x_path, os.O_CREAT | os.O_WRONLY | os.O_NOFOLLOW, 0700)
        os.write(xstartup, "#!/bin/bash\n")
        os.close(xstartup)
        lock_stat = os.lstat(x_path)
      except OSError:
        raise MaxwellError("Unable to create '%s'." % x_path)

    # check that it has the expected permissions and ownership
    # check that we are the owner and that others can't write
    if lock_stat[stat.ST_MODE] & stat.S_IWOTH:
      raise MaxwellError("'%s' has unsafe permissions.  Remove write permissions for others"
        % x_path)

    usr_id = lock_stat[stat.ST_UID]
    if usr_id != os.geteuid():
      raise MaxwellError("'%s' has incorrect owner: %s" % (x_path, usr_id))

  def read_passwd(self):
    """VNC password is 8 bytes, we want the version encrypted for VNC, not the
    encoded version for web use
    """
    self.vncpass = sys.stdin.read(8)
    return

  def stunnel(self):
    """We handle tunnels and forwards here, to make the inside of containers visible to the outside.
    Containers are mapped to port ranges using the dispnum (a.k.a. CTID a.k.a. veid).
    """
    in_port = self.veid + self.k["STUNNEL_PORTS"] # e.g., 4000 + display
    remote = '%s:%d' % (self.veaddr, self.k["PORTBASE"])
    # kill anything listening on the stunnel port; should have been killed when stopping container
    # to avoid race condition with TCP_WAIT state.
    p = subprocess.Popen(['fuser', '-n', 'tcp', '%d' % in_port, '-k', '-9'], stderr=subprocess.PIPE, stdout=subprocess.PIPE)
    # retrieve but ignore error message which is generated if there was no process to kill
    (stdout, stderr) = p.communicate()
    if p.returncode == 0:
      # a process was killed, wait a bit for TCP_WAIT state to clear
      time.sleep(1)
    if self.k["TUNNEL_MODE"][:5] == 'socat':
      args = ["socat"]
      args.append("OPENSSL-LISTEN:%d,cert=%s,fork,verify=0" % (in_port, self.k["PEM_PATH"]))
      args.append("tcp4:%s" % remote)
      if len(self.k["TUNNEL_MODE"]) > 5:
        args += ["-d", "-d", "-d"]
1046 1047
      if DEBUG:
        log('DEBUG: ' + " ".join(args))
Pascal Meunier's avatar
Pascal Meunier committed
1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082
      process = subprocess.Popen(
        args,
        stdout = subprocess.PIPE,
        stderr = subprocess.PIPE
      )
      (stdout, stderr) = process.communicate()
      if process.returncode != 0:
        log("Can't start ssl socat (it's probably already running): %s" % stderr)

      if VERBOSE:
        log("forwarder started: %s" % stdout)

    else:
      if self.k["TUNNEL_MODE"] == 'stunnel4':
        # stunnel4 will read configuration from a pipe.
        # reading from stdin causes the first connection to fail
        # we're going to create a pipe and fork because Popen is too limited
        # file descriptors r, w for reading and writing
        r, w = os.pipe() 
        processid = os.fork()
        if processid:
          # This is the parent process
          # note: the parent can't exec otherwise the webserver will think the container is done starting
          os.close(r)
          w = os.fdopen(w, 'w')
          w.write("cert = %s\n" % self.k["PEM_PATH"])
          w.write("accept = %d\n" % in_port)
          w.write("connect = %s\n" % remote)
          w.write("debug = 3\n")
          # RedHat has FIPS version, this errors out on Debian.
          w.write("fips=no\n")
          w.write("output=/var/log/stunnel\n")
          w.write("[stunnel3]\n")
          w.close()
          (pid, status) = os.wait()
1083 1084
          if status == 0 and VERBOSE:
            log("VERBOSE: stunnel4 os.wait pid=%d, status=%d." %(pid, status))
Pascal Meunier's avatar
Pascal Meunier committed
1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103
          else:
            raise MaxwellError("stunnel error! pid=%d, status=%d." %(pid, status))
        else:
          # This is the child process, will read from r
          os.close(w)
          os.execl("/usr/bin/stunnel", "/usr/bin/stunnel", "-fd", "%d" % r)
          raise MaxwellError("unable to execute /usr/bin/stunnel")
      else:
        # stunnel3 for Debian
        # status = os.system("stunnel -d %d -r %s:%d -p %s" %
        #         (4000+self.disp, self.veaddr, 5000,  self.k["PEM_PATH"]))
        # -d: daemon mode
        # -r [host:]port    connect to remote service
        # , '-D', '7' to increase debug level to 7
        args = ["stunnel", '-D', '4', '-d', str(in_port), '-r', remote, '-p',  self.k["PEM_PATH"]]
        log(" ".join(args))
        subprocess.check_call(args)

      if VERBOSE:
1104
        log("VERBOSE: stunnel started for %d" % self.veid)
Pascal Meunier's avatar
Pascal Meunier committed
1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116

    # Start a forwarder to make the display look external.
    # This is only for backward-compatibility.
    #os.system("socat tcp4-listen:%d,fork,reuseaddr,linger=0 tcp4:%s:5000 > /dev/null 2>&1 < /dev/null &" % (5000+ self.veid, self.veaddr))

  def delete_confs(self):
    """Get rid of these links if they exist."""
    for ext in ['conf', 'mount', 'umount']:
      try:
        os.unlink("%s/%d.%s" % (self.k["VZ_CONF_PATH"], self.veid, ext))
      except EnvironmentError:
        if DEBUG:
1117
          log("DEBUG: File %s/%d.%s was already deleted or missing" %
Pascal Meunier's avatar
Pascal Meunier committed
1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130
            (self.k["VZ_CONF_PATH"], self.veid, ext))

    # stop quotas if they are already running
    # vzquota off 194
    # vzquota drop 194
    # this operation can fail if the operations are still ongoing; that's fine.
    subprocess.call(['/usr/sbin/vzquota', 'off', '%d' % self.veid], stderr=open('/dev/null', 'w'))
    # drop safely removes the quota file -- this file can cause problems,
    # e.g., when container template is changed
    subprocess.call(['/usr/sbin/vzquota', 'drop', '%d' % self.veid], stderr=open('/dev/null', 'w'))

  def create_confs(self):
    """In directory /etc/vz/conf, create the symlinks to the mount, unmount scripts and
1131 1132
    the configuration file.  The mount script is called when starting the VE (container).
    Ignore errors if symlinks already exist."""
Pascal Meunier's avatar
Pascal Meunier committed
1133 1134 1135
    try:
      os.symlink("%s/%s" % (self.k["VZ_CONF_PATH"], self.k["OVZ_SESSION_CONF"]),
        "%s/%d.conf" % (self.k["VZ_CONF_PATH"], self.veid))
1136 1137 1138
    except EnvironmentError:
      pass
    try:
Pascal Meunier's avatar
Pascal Meunier committed
1139 1140
      os.symlink("%s/%s" % (self.k["VZ_CONF_PATH"], self.k["OVZ_SESSION_MOUNT"]),
        "%s/%d.mount" % (self.k["VZ_CONF_PATH"], self.veid))
1141 1142 1143
    except EnvironmentError:
      pass
    try:
Pascal Meunier's avatar
Pascal Meunier committed
1144 1145 1146
      os.symlink("%s/%s" % (self.k["VZ_CONF_PATH"], self.k["OVZ_SESSION_UMOUNT"]),
        "%s/%d.umount" % (self.k["VZ_CONF_PATH"], self.veid))
    except EnvironmentError:
1147
      pass
Pascal Meunier's avatar
Pascal Meunier committed
1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172

  def start_filexfer(self):
    """Start a socat forwarder for filexfer.  We never kill it.
       If we can't start one, that means there's already one running."""
    port = self.veid + self.k["FILEXFER_PORTS"]
    os.system("socat tcp4-listen:%d,fork,reuseaddr,linger=0 tcp4:%s:%d > /dev/null 2>&1 &"
      % (port, self.veaddr, port))

  def start(self, geom):
    """ start a container.
    Have /usr be symlink at the beginning (from setup_template), then remove it to put a mount
    Setup a lock directory that will be erased by the start process when it's done
    this functionality appears to be duplicated by the .mount scripts in /etc/vz/conf.
    We wait for the lock to be removed, to indicate that the mount script has finished.  This is
    not an access lock.
    """
    lock_dir = "%s/lock/mount.%d.lock" % (self.k["VZ_PATH"], self.veid)
    if not os.path.exists(lock_dir):
      os.mkdir(lock_dir)
      if VERBOSE:
        log("Created %s" % lock_dir)
    else:
      if VERBOSE:
        log("Already existed: %s" % lock_dir)
    start_time = time.time()
1173 1174 1175 1176 1177
    if DEBUG:
      # extremely verbose
      args = ["vzctl", "--verbose", "start", str(self.veid)]
    else:
      args = ["vzctl", "start", str(self.veid)]
Pascal Meunier's avatar
Pascal Meunier committed
1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256
    p = subprocess.Popen(args, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
    log_subprocess(p)
    if p.returncode != 0:
      raise MaxwellError("Can't start container '%d'" % (self.veid))
    # now sleep until the lock is gone
    while os.path.exists(lock_dir):
      time.sleep(1)
      if time.time() - start_time > 60:
        raise MaxwellError("Timed out waiting for container to start")
    end_time = time.time()
    # log how long we waited
    log ("vzctl start time: %f seconds" % (end_time - start_time))

    # replace the symlink with a mount
    os.unlink(self.vz_private_path + "/usr")

    # If vz/root mount is done after the call to start, without a symlink in place, we get:
    # bash: line 318: awk: command not found
    # ERROR: Can't change file /etc/hosts

    # If vz/root mount is done before the call to start, we get:
    # error 32
    # mount: special device /vz/root/257/.root/usr does not exist
    #
    # if we try private instead of root, we get:
    # mount: special device /vz/private/261/.root/usr does not exist
    #
    # symlink can't be left alone due to bug in gcc;  mounting is needed.
    #
    # check mount point exists or create it
    usr_mnt = self.vz_root_path + "/usr"
    if not os.path.exists(usr_mnt):
      os.mkdir(usr_mnt)

    # mount --bind olddir newdir
    # -n: Mount without writing in /etc/mtab.
    args = ["/bin/mount", "-n", "--bind", self.vz_root_path + "/.root/usr", usr_mnt]
    p = subprocess.Popen(args, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
    log_subprocess(p)
    if p.returncode != 0:
      raise MaxwellError("Can't bind mount .root/usr in '%d'" % (self.veid))

    if VERBOSE:
      log("vzctl exec2 %d %s %s 0 %s" %\
          (self.veid, self.k["INTERNAL_PATH"]+'startxvnc', self.veaddr, geom))
    args = ["vzctl", "exec2", str(self.veid)]
    args += [self.k["INTERNAL_PATH"] + 'startxvnc', self.veaddr, '0', geom]
    process = subprocess.Popen(
      args,
      stdin = subprocess.PIPE,
      stdout = subprocess.PIPE,
      stderr = subprocess.PIPE
    )
    (stdout, stderr) = process.communicate(self.vncpass)
    if process.returncode != 0:
      raise MaxwellError("Unable to start internal Xvnc server: %s%s" %(stdout, stderr))
    elif VERBOSE:
      log(stdout)
      end_time3 = time.time()
      # log how long we waited
      log ("startxvnc call took: %f seconds" % (end_time3 - end_time))

  def resize(self, geometry):
    """Change XVNC geometry on the fly, after the container has started."""
    (width, height) = geometry.split('x')
    args = ["vzctl", "exec2", str(self.veid), self.k["INTERNAL_PATH"] + 'hzvncresize']
    args += ['-a', '/var/run/Xvnc/passwd-%s:0' % self.veaddr, width, height ]
    process = subprocess.Popen(
      args,
      stdin = subprocess.PIPE,
      stdout = subprocess.PIPE,
      stderr = subprocess.PIPE
    )
    (stdout, stderr) = process.communicate()
    if process.returncode != 0:
      raise MaxwellError("Unable to change Xvnc geometry: %s%s" %(stdout, stderr))

  def setup_template(self):
    """Setup symlinks and mount points for OpenVZ container.
Pascal Meunier's avatar
Pascal Meunier committed
1257 1258 1259 1260
        usr is a temporary symlink.  It needs to be a symlink initially and later 
        we replace it with a mount point.  The mount point is needed 
        because some versions of gcc don't work with a symlink.
        Doing the mount point without first doing the symlink sometimes generates these:
Pascal Meunier's avatar
Pascal Meunier committed
1261 1262 1263
      # bash: line 318: awk: command not found
      #  ERROR: Can't change file /etc/hosts
    """
1264 1265
    os.makedirs(self.vz_root_path)
    os.chmod(self.vz_root_path, 0755)
Pascal Meunier's avatar
Pascal Meunier committed
1266 1267
    if VERBOSE:
      log("created directory " + self.vz_root_path)
1268 1269
    os.makedirs(self.vz_private_path)
    os.chmod(self.vz_private_path, 0755)
Pascal Meunier's avatar
Pascal Meunier committed
1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285
    if VERBOSE:
      log("created directory " + self.vz_private_path)
    # for link in ['bin', 'lib', 'sbin', 'lib64', 'usr']: equivalent to "template copy -a" method
    # also link 'emul', 'lib32' to support 32-bit binaries
    for link in ['bin', 'lib', 'sbin', 'emul', 'lib32', 'lib64', 'usr', 'opt']:
      path = self.vz_private_path + "/" + link
      # treating usr as a mount point can generate these errors:
      # bash: line 318: awk: command not found
      #  ERROR: Can't change file /etc/hosts
      if os.path.lexists(path):
        log("%s already exists!" % path)
      else:
        os.symlink(".root/%s" % link, path)

    for vzdir in ['.root', 'home', 'mnt', 'proc', 'sys']:
      os.mkdir(self.vz_private_path + "/" + vzdir)
1286 1287
      os.chmod(self.vz_private_path + "/" + vzdir, 0755)

Pascal Meunier's avatar
Pascal Meunier committed
1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337
    if VERBOSE:
      log("template setup")

  def __delete_root(self):
    """Delete container root path.  If it doesn't exist, it will fail the directory test"""
    if os.path.isdir(self.vz_root_path):
      os.rmdir(self.vz_root_path)
      # what if directory isn't empty?

  def __log_status(self):
    """log the status of the container if in VERBOSE mode"""
    if VERBOSE:
      log(self.get_status())

  def get_status(self):
    """Obtain the status of this container"""
    args = ['vzctl', 'status', str(self.veid)]
    p = subprocess.Popen(args, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
    (stdout, stderr) = p.communicate()
    if p.returncode != 0:
      raise MaxwellError("Can't get status of container '%d': %s" % (self.veid, stderr))
    return str(stdout)

  def __halt(self):
    """ Hard halt for all processes in the container.  Does not wait for anything."""
    args = ['vzctl', 'exec', str(self.veid), 'halt', '-nf']
    process = subprocess.Popen(
      args,
      stdout = subprocess.PIPE,
      stderr = subprocess.PIPE
    )
    (stdout, stderr) = process.communicate()
    if process.returncode == 14:
      # Container configuration file vps.conf(5) not found
      # try to fix it otherwise VE can't be shut down!
      self.create_confs()
      subprocess.check_call(args)
    if process.returncode != 0:
      log("Unable to halt VE: %s%s" %(stdout, stderr))

  def wait_unlock(self):
    """Allow the caller to know when OpenVZ is done starting or stopping a container"""
    attempt = 0
    while os.path.exists("/vz/lock/%d" % self.veid):
      time.sleep(10)
      attempt += 1
      if attempt > 50:
        raise MaxwellError("Unable to get lock on VE: %d" %(self.veid))

  def stop(self):
1338
    """# Stops  and  unmounts  a  container. 
1339 1340 1341
    Use '--fast' option so OpenVZ leaves mounts alone for our .umount script to handle. 
    Otherwise OpenVZ makes /vz/root/VEID read-only"""
    args = ['vzctl', 'stop', str(self.veid), '--fast']
1342
    process = subprocess.Popen(args, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
Pascal Meunier's avatar
Pascal Meunier committed
1343 1344 1345 1346 1347 1348 1349 1350
    (stdout, stderr) = process.communicate()
    if process.returncode == 14:
      # Container configuration file vps.conf(5) not found
      # try to fix it otherwise VE can't be shut down!
      self.create_confs()
      p = subprocess.Popen(args, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
      log_subprocess(p)
      return
1351
    if process.returncode == 3:
1352
      # Error in waitpid(12345): No child processes
1353 1354
      # try to umount instead
      self.umount()
1355 1356
      if VERBOSE:
        log("ignoring error 3 in 'vzctl stop' : %s%s exit code %d"
1357 1358
        % (stdout, stderr, process.returncode))
    elif process.returncode != 0:
Pascal Meunier's avatar
Pascal Meunier committed
1359 1360
      raise MaxwellError("'vzctl stop' output: %s%s exit code %d"
        % (stdout, stderr, process.returncode))
1361 1362
    else:
      log(stdout)
1363 1364 1365 1366
      # delete the directory only if successful
      # We needed to wait until now because "vzctl stop" expects it.
      if os.path.isdir(self.vz_root_path):
        os.rmdir(self.vz_root_path)
Pascal Meunier's avatar
Pascal Meunier committed
1367 1368

  def __vzproccount(self):
1369 1370 1371
    """Read /proc/vz/veinfo to get the number of processes in the container.
    Each line presents a running Container in the <CT_ID> <reserved> <number_of_processes> <IP_address> ... format: 
    """
Pascal Meunier's avatar
Pascal Meunier committed
1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395
    try:
      f = open("/proc/vz/veinfo")
    except EnvironmentError:
      log("vzproccount: can't open veinfo.")
      # possibly stopped
      return 0
    while 1:
      line = f.readline()
      if line == "":
        if False:
          log("End of file /proc/vz/veinfo.")
        return 0
      arr = line.split()
      # expecting something like "         29     0     2      10.26.0.29"
      if len(arr) != 4:
        continue
      if arr[0] == str(self.veid):
        #log("vzproccount is %s" % arr[2])
        try:
          return int(arr[2])
        except ValueError:
          return 0

  def stop_submit_local(self):
1396 1397
    """check for submit --local.  If it's there, give it SIGINT and wait.
    Called by maxwell_service before calling killall."""
Pascal Meunier's avatar
Pascal Meunier committed
1398 1399 1400 1401 1402
    # kill stunnel and log any errors as it should still be running
    in_port = self.veid + self.k["STUNNEL_PORTS"] # e.g., 4000 + display
    p = subprocess.Popen(['fuser', '-n', 'tcp', '%d' % in_port, '-k', '-9'], stderr=subprocess.PIPE, stdout=subprocess.PIPE)
    log_subprocess(p)
    check_submit_args = ['vzctl', 'exec', str(self.veid), '/usr/bin/pgrep', '-f', '\"submit --local\"']
1403 1404 1405
    p = subprocess.Popen(check_submit_args, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
    (stdout, stderr) = p.communicate()
    rc = p.returncode
1406 1407 1408 1409 1410
    if DEBUG:
      if len(stderr) > 0:
        log("DEBUG: error finding submit pid: " + stderr.rstrip())
      if len(stdout) > 0:
        log("DEBUG: submit pid was: " + stdout.rstrip())
Pascal Meunier's avatar
Pascal Meunier committed
1411
    if rc == 0:
1412 1413 1414 1415
      log("Signal submit --local to exit")
      pkill_args = ['vzctl', 'exec', str(self.veid), 'pkill', '-15', '-f', '\"submit --local\"']
      p = subprocess.Popen(pkill_args, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
      (stdout, stderr) = p.communicate()
Pascal Meunier's avatar
Pascal Meunier committed
1416
      attempt = 0
1417 1418 1419 1420
      p = subprocess.Popen(check_submit_args, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
      # discard output
      p.communicate()
      rc = p.returncode
Pascal Meunier's avatar
Pascal Meunier committed
1421 1422
      while rc == 0 and (attempt < 10):
        attempt += 1
1423
        time.sleep(20) # give time for submit to exit
1424 1425 1426 1427
        p = subprocess.Popen(check_submit_args, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
        # discard output
        p.communicate()
        rc = p.returncode
Pascal Meunier's avatar
Pascal Meunier committed
1428 1429
      if attempt > 9:
        log("submit --local didn't exit!")
1430
    elif DEBUG:
1431
      log("DEBUG stop_submit_local: exit code %d.  Running processes:" % rc)
1432
      args= ['vzctl', 'exec', str(self.veid), 'ps aux']
1433
      p = subprocess.Popen(args, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
1434
      log_subprocess(p)
Pascal Meunier's avatar
Pascal Meunier committed
1435 1436

  def killall(self):
1437
    """Kill processes other than init running in the VPS.
1438
    Start with milder signals.  Wait as long as the number of processes goes down.
1439
    More recent kernels count kthreadd and khelper processes that are unkillable.
1440

1441 1442 1443 1444 1445
    Called by maxwell_service in 2 cases: after a tool is done running, and when receiving the stopvnc command."""
    status = self.get_status()
    if status.find("running") == -1:
      if DEBUG:
        log("DEBUG: killall: container was not running")
1446 1447
      # container isn't running, make sure root and private areas are unmounted.
      self.umount()
1448
      
Pascal Meunier's avatar
Pascal Meunier committed
1449 1450
    for sig in [1, 2, 15, 9]:
      pcount = self.__vzproccount()
1451
      # send all signals to avoid issues with unmounting
1452 1453
      if pcount == 0:
        break
Pascal Meunier's avatar
Pascal Meunier committed
1454 1455 1456 1457 1458 1459 1460 1461
      log("Killing %d processes in veid %d with signal %d" % (self.__vzproccount(), self.veid, sig))
      # -1 indicates all processes except the kill process itself and init.
      args = ['vzctl', 'exec', str(self.veid), 'kill -%d -1' % sig]
      p = subprocess.Popen(args, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
      log_subprocess(p)
      # Wait as long as the number of processes keeps going down
      ccount = 0
      tries = 0
1462
      while ccount < pcount and tries < 100:
1463
        time.sleep(2) # give time for processes to exit, otherwise we try again too early
Pascal Meunier's avatar
Pascal Meunier committed
1464 1465 1466 1467
        pcount = ccount
        ccount = self.__vzproccount()
        tries += 1
        # log("attempt %d, signal %d, count is %d" % (tries, sig, ccount))
1468
      if tries == 100:
Pascal Meunier's avatar
Pascal Meunier committed
1469
        log("timeout waiting for processes to exit from signal %d" % (sig))
1470
    # wait because in some edge cases the container shuts down
Pascal Meunier's avatar
Pascal Meunier committed
1471 1472
    attempt = 0
    while self.__vzproccount() > 1 and (attempt < 5):
1473 1474
      if DEBUG:
        log("DEBUG: killall: final wait")
Pascal Meunier's avatar
Pascal Meunier committed
1475
      attempt += 1
1476
      time.sleep(1)
Pascal Meunier's avatar
Pascal Meunier committed
1477

1478
class ContainerVZ7(Container):
1479
  """Modified for OpenVZ 7 compatibility"""
1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493
  def __init__(self, disp, machine_number, overrides={}):
    # OpenVZ 7: Seems to have 2 folder hierarchies it can work with.
    # one mounts /vz/private/veid/fs to /vz/root/veid, and has more folders and files under /vz/private/veid/
    # the other maintains compatibility with /vz/private/veid being mounted directly
    # in this class we use the compatibility option
    # if we were converting to the new format we'd do:
    # self.vz_private_path = "%s/private/%d/fs" % (self.k["VZ_PATH"], self.veid)
    # but that's not enough.  Not sure what is the trigger between vzctl using one or the other
    Container.__init__(self, disp, machine_number, overrides)
    
  def create_xstartup(self):
    """LINT.
    This function created an "xstartup" file for VNC.  Does not appear needed anymore.    
    """
1494
    pass
1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515

  def delete_confs(self):
    """Get rid of these links if they exist."""
    for ext in ['conf', 'mount', 'umount']:
      try:
        os.unlink("%s/%d.%s" % (self.k["VZ_CONF_PATH"], self.veid, ext))
      except EnvironmentError:
        if DEBUG:
          log("File %s/%d.%s was already deleted or missing" %
            (self.k["VZ_CONF_PATH"], self.veid, ext))
    # OpenVZ 7 has withdrawn filesystem quota support from legacy simfs, vzquota calls removed

  def create_confs(self):
    """In directory /etc/vz/conf, create a symlink to the configuration file.  
    Under OpenVZ 7, we don't create symlinks for the mount and .umount scripts
    because vps.mount and vps.umount are called, regardless of container ID. """
    try:
      os.symlink("%s/%s" % (self.k["VZ_CONF_PATH"], self.k["OVZ_SESSION_CONF"]),
        "%s/%d.conf" % (self.k["VZ_CONF_PATH"], self.veid))
    except EnvironmentError:
      raise MaxwellError("Unable to create OpenVZ symlinks")
1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532

class ContainerDocker(Container):
  """Modified to use Docker.
  We run 2 docker containers per tool session.  
  One runs services like VNC, the other runs the tool.  The advantage of doing it this way is we can uncouple the VNC server from the tool -- i.e., use different templates, possibly running different OS versions.  The services could use more up-to-date packages, so vulnerability patching could be done without needing to upgrade tools.  Container names would be <self.veid>-services and <self.veid>-tool.  The call to start a standby session will start the <self.veid>-services container.  We'll keep only one standby session when using Docker.
  """

  def __init__(self, disp, machine_number, overrides={}):
    # machine_number: ignored (lint)
    self.k = CONTAINER_K
    self.k.update(overrides)
    self.disp = disp
    self.veid = disp
    self.vncpass = None
    # mounts:  array to record list of bind mounts to make when starting container
    self.mounts = []
    self.deleted = False
1533 1534 1535 1536
    if 'IONHELPER' in self.k:
      self.ionhelper = True
    else:
      self.ionhelper = False
1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594
    if disp < 1:
      raise MaxwellError("Container ID must be at least 1")
    #
    # we use /22 networks
    if disp > 1021:
      # container count starts at 1;  we want container #1 to use the .1 address
      # 1023th IP is broadcast address (3*256 + 255)
      # 1022th IP is the Docker bridge's IP (3*256 + 254) because 
      raise MaxwellError("container ID %d too high, maximum is 1021" % disp)

    # tools network doesn't offer any services, by default doesn't connect to internet
    self.toolnet_name = 'toolnet'

    # service network provides X server at known IP address for stunnel to connect to it
    # without IP collision in .Xauthority file
    self.servicenet_name = 'servicenet'
    # service net CIDR provided by self.k["PRIVATE_NET"] and machine_number calculation
    
    self.toolnet_CIDR = self.k["DOCKERTOOL_NET"] + '/22'
    gwbytes = self.k["DOCKERTOOL_NET"].split('.')
    gateway = gwbytes[0:2] + [str(int(gwbytes[2]) + 3), '254']
    self.toolnet_gateway = ".".join(gateway)
    IPa = gwbytes[0:2] + [str(int(gwbytes[2]) + disp/256), str(disp %256)]
    self.tool_container_IP = ".".join(IPa)
    
    self.servicenet_CIDR = self.k["DOCKERSERVICE_NET"] + '/22'
    gwbytes = self.k["DOCKERSERVICE_NET"].split('.')
    gateway = gwbytes[0:2] + [str(int(gwbytes[2]) + 3), '254']
    self.servicenet_gateway = ".".join(gateway) 
    IPa = gwbytes[0:2] + [str(int(gwbytes[2]) + disp/256), str(disp %256)]
    self.services_container_IP = ".".join(IPa)
    
    # for inherited stunnel function
    self.veaddr = self.services_container_IP

  def resize(self, geometry):
    """Change XVNC geometry on the fly, after the container has started."""
    (width, height) = geometry.split('x')
    args = ["docker", "exec"]
    args += ['-e', 'DISPLAY=%s:0' % self.services_container_IP]
    args += ['%d.services' % self.veid]
    args += [self.k["INTERNAL_PATH"] + 'hzvncresize', '-a', '/var/run/Xvnc/passwd-%s:0' % self.services_container_IP, width, height ]
    process = subprocess.Popen(
      args,
      stdin = subprocess.PIPE,
      stdout = subprocess.PIPE,
      stderr = subprocess.PIPE
    )
    (stdout, stderr) = process.communicate()
    if process.returncode != 0:
      if DEBUG:
        log(" ".join(args))
      raise MaxwellError("Unable to change Xvnc geometry: %s%s" %(stdout, stderr))
    elif VERBOSE:
      log(stdout)

  def screenshot(self, user, sessionid):
    """Support display of session screenshots for app UI.  On error, do not produce an exception."""
1595
    account = make_User_account(user, self.k)
1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608
    destination = "%s/data/sessions/%s/screenshot.png" % (account.homedir, sessionid)
    if os.path.isdir("%s/data/sessions/%s" % (account.homedir, sessionid)):
      args = ["docker", "exec"]
      args += ['-e', 'DISPLAY=%s:0' % self.services_container_IP]
      args += ['--user', user, '%d.tool' % self.veid]
      args += ["/usr/bin/screenshot", destination]
      p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
      log_subprocess(p)

  def create_xstartup(self):
    """LINT.  This function created an "xstartup" file for VNC.  Does not appear needed anymore.    
    """
    pass