From 63b0e2ba44898c46a46aff03c9c02e80ab9724b9 Mon Sep 17 00:00:00 2001 From: PsychoMario Date: Thu, 16 Apr 2015 18:24:12 +0100 Subject: [PATCH 01/39] Added static assignment support for #27 --- example-leases.json | 15 +++++++++++++++ pypxe/dhcp.py | 22 +++++++++++++++++----- 2 files changed, 32 insertions(+), 5 deletions(-) create mode 100644 example-leases.json diff --git a/example-leases.json b/example-leases.json new file mode 100644 index 0000000..7b82aa2 --- /dev/null +++ b/example-leases.json @@ -0,0 +1,15 @@ +{ + "dhcp": { + "binding": { + "00:26:B9:A0:97:29": { + "dns": [ + "8.8.8.8", + "8.8.4.4" + ], + "ipaddr": "192.168.2.123", + "router": "192.168.2.5", + "subnet": "255.255.255.0" + } + } + } +} diff --git a/pypxe/dhcp.py b/pypxe/dhcp.py index 89156bc..13135ad 100644 --- a/pypxe/dhcp.py +++ b/pypxe/dhcp.py @@ -42,6 +42,7 @@ def __init__(self, **server_settings): self.http = server_settings.get('use_http', False) self.mode_proxy = server_settings.get('mode_proxy', False) # ProxyDHCP mode self.mode_debug = server_settings.get('mode_debug', False) # debug mode + self.static_config = serverSettings.get('static_config', dict()) self.magic = struct.pack('!I', 0x63825363) # magic cookie self.logger = server_settings.get('logger', None) @@ -88,6 +89,12 @@ def __init__(self, **server_settings): # key is MAC self.leases = defaultdict(lambda: {'ip': '', 'expire': 0, 'ipxe': self.ipxe}) + def getNamespacedStatic(self, path, fallback = {}): + statics = self.static_config + for child in path.split('.'): + statics = statics.get(child, {}) + return statics if statics else fallback + def next_ip(self): ''' This method returns the next unleased IP from range; @@ -164,7 +171,8 @@ def craft_header(self, message): if self.leases[client_mac]['ip']: # OFFER offer = self.leases[client_mac]['ip'] else: # ACK - offer = self.next_ip() + offer = self.getNamespacedStatic('dhcp.binding.{}.ipaddr'.format(self.printMAC(clientmac))) + offer = offer if offer else self.next_ip() self.leases[client_mac]['ip'] = offer self.leases[client_mac]['expire'] = time() + 86400 self.logger.debug('New Assignment - MAC: {0} -> IP: {1}'.format(self.print_mac(client_mac), self.leases[client_mac]['ip'])) @@ -196,10 +204,14 @@ def craft_options(self, opt53, client_mac): response = self.tlv_encode(53, chr(opt53)) # message type, OFFER response += self.tlv_encode(54, socket.inet_aton(self.ip)) # DHCP Server if not self.mode_proxy: - response += self.tlv_encode(1, socket.inet_aton(self.subnet_mask)) # Subnet Mask - response += self.tlv_encode(3, socket.inet_aton(self.router)) # Router - response += self.tlv_encode(6, socket.inet_aton(self.dns_server)) # DNS - response += self.tlv_encode(51, struct.pack('!I', 86400)) # lease time + subnetmask = self.getNamespacedStatic('dhcp.binding.{}.subnet'.format(self.printMAC(clientmac)), self.subnetmask) + response += self.tlvEncode(1, socket.inet_aton(subnetmask)) #SubnetMask + router = self.getNamespacedStatic('dhcp.binding.{}.router'.format(self.printMAC(clientmac)), self.router) + response += self.tlvEncode(3, socket.inet_aton(router)) #Router + dnsserver = self.getNamespacedStatic('dhcp.binding.{}.dns'.format(self.printMAC(clientmac)), [self.dnsserver]) + dnsserver = ''.join([socket.inet_aton(i) for i in dnsserver]) + response += self.tlvEncode(6, dnsserver) + response += self.tlvEncode(51, struct.pack('!I', 86400)) #lease time # TFTP Server OR HTTP Server; if iPXE, need both response += self.tlv_encode(66, self.file_server) From 70b7691b0a6a325cafd77342ce62f1e4d57d0ad6 Mon Sep 17 00:00:00 2001 From: PsychoMario Date: Sun, 10 May 2015 23:13:30 +0100 Subject: [PATCH 02/39] added static-config CLI handling --- pypxe-server.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/pypxe-server.py b/pypxe-server.py index 066feba..d269dd7 100755 --- a/pypxe-server.py +++ b/pypxe-server.py @@ -45,6 +45,7 @@ def parse_cli_arguments(): parser.add_argument('--no-tftp', action = 'store_false', dest = 'USE_TFTP', help = 'Disable built-in TFTP server, by default it is enabled', default = SETTINGS['USE_TFTP']) parser.add_argument('--debug', action = 'store', dest = 'MODE_DEBUG', help = 'Comma Seperated (http,tftp,dhcp). Adds verbosity to the selected services while they run. Use \'all\' for enabling debug on all services', default = SETTINGS['MODE_DEBUG']) parser.add_argument('--config', action = 'store', dest = 'JSON_CONFIG', help = 'Configure from a JSON file rather than the command line', default = '') + parser.add_argument('--static-config', action = 'store', dest = 'STATIC_CONFIG', help = 'Configure leases from a json file rather than the command line', default = '') parser.add_argument('--syslog', action = 'store', dest = 'SYSLOG_SERVER', help = 'Syslog server', default = SETTINGS['SYSLOG_SERVER']) parser.add_argument('--syslog-port', action = 'store', dest = 'SYSLOG_PORT', help = 'Syslog server port', default = SETTINGS['SYSLOG_PORT']) @@ -92,6 +93,21 @@ def parse_cli_arguments(): SETTINGS.update(loaded_config) # update settings with JSON config args = parse_cli_arguments() # re-parse, CLI options take precedence + # ideally this would be in dhcp itself, but the chroot below *probably* + # breaks the ability to open the config file. + if args.STATIC_CONFIG: + try: + static_config = open(args.STATIC_CONFIG, 'rb') + except IOError: + sys.exit("Failed to open {0}".format(args.STATIC_CONFIG)) + try: + loaded_statics = json.load(static_config) + static_config.close() + except ValueError: + sys.exit("{0} does not contain valid json".format(args.STATIC_CONFIG)) + else: + loaded_statics = dict() + # setup main logger sys_logger = logging.getLogger('PyPXE') if args.SYSLOG_SERVER: @@ -167,7 +183,8 @@ def parse_cli_arguments(): use_http = args.USE_HTTP, mode_proxy = args.DHCP_MODE_PROXY, mode_debug = ('dhcp' in args.MODE_DEBUG.lower() or 'all' in args.MODE_DEBUG.lower()), - logger = dhcp_logger) + logger = dhcp_logger, + static_config = loaded_statics) dhcpd = threading.Thread(target = dhcp_server.listen) dhcpd.daemon = True dhcpd.start() From ddc32b6087bae121457aaf9174edd65cee993a2a Mon Sep 17 00:00:00 2001 From: PsychoMario Date: Sun, 10 May 2015 23:20:53 +0100 Subject: [PATCH 03/39] fix #79 --- pypxe/dhcp.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/pypxe/dhcp.py b/pypxe/dhcp.py index 13135ad..358d58f 100644 --- a/pypxe/dhcp.py +++ b/pypxe/dhcp.py @@ -42,7 +42,7 @@ def __init__(self, **server_settings): self.http = server_settings.get('use_http', False) self.mode_proxy = server_settings.get('mode_proxy', False) # ProxyDHCP mode self.mode_debug = server_settings.get('mode_debug', False) # debug mode - self.static_config = serverSettings.get('static_config', dict()) + self.static_config = server_settings.get('static_config', dict()) self.magic = struct.pack('!I', 0x63825363) # magic cookie self.logger = server_settings.get('logger', None) @@ -219,8 +219,8 @@ def craft_options(self, opt53, client_mac): # file_name null terminated if not self.ipxe or not self.leases[client_mac]['ipxe']: # http://www.syslinux.org/wiki/index.php/PXELINUX#UEFI - if 93 in self.options and not self.force_file_name: - [arch] = struct.unpack("!H", self.options[93][0]) + if 93 in self.leases[client_mac]["options"] and not self.force_file_name: + [arch] = struct.unpack("!H", self.leases[client_mac]["options"][93][0]) if arch == 0: # BIOS/default response += self.tlv_encode(67, "pxelinux.0" + chr(0)) elif arch == 6: # EFI IA32 @@ -275,9 +275,9 @@ def dhcp_ack(self, message): self.logger.debug('<--END RESPONSE-->') self.sock.sendto(response, (self.broadcast, 68)) - def validate_req(self): + def validate_req(self, client_mac): # client request is valid only if contains Vendor-Class = PXEClient - if 60 in self.options and 'PXEClient' in self.options[60][0]: + if 60 in self.leases[client_mac]["options"] and 'PXEClient' in self.leases[client_mac]["options"][60][0]: self.logger.debug('PXE client request received') return True if self.mode_debug: @@ -293,14 +293,14 @@ def listen(self): self.logger.debug('<--BEGIN MESSAGE-->') self.logger.debug('{0}'.format(repr(message))) self.logger.debug('<--END MESSAGE-->') - self.options = self.tlv_parse(message[240:]) + self.leases[client_mac]["options"] = self.tlv_parse(message[240:]) self.logger.debug('Parsed received options') self.logger.debug('<--BEGIN OPTIONS-->') - self.logger.debug('{0}'.format(repr(self.options))) + self.logger.debug('{0}'.format(repr(self.leases[client_mac]["options"]))) self.logger.debug('<--END OPTIONS-->') - if not self.validate_req(): + if not self.validate_req(client_mac): continue - type = ord(self.options[53][0]) # see RFC2131, page 10 + type = ord(self.leases[client_mac]["options"][53][0]) # see RFC2131, page 10 if type == 1: self.logger.debug('Received DHCPOFFER') try: From bd89708da346cef53114c75dc0cd4e6163c8f3f8 Mon Sep 17 00:00:00 2001 From: PsychoMario Date: Mon, 11 May 2015 00:16:14 +0100 Subject: [PATCH 04/39] added DHCP whitelist --- pypxe-server.py | 2 ++ pypxe/dhcp.py | 6 +++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/pypxe-server.py b/pypxe-server.py index d269dd7..5d92355 100755 --- a/pypxe-server.py +++ b/pypxe-server.py @@ -62,6 +62,7 @@ def parse_cli_arguments(): parser.add_argument('-d', '--dhcp-dns', action = 'store', dest = 'DHCP_DNS', help = 'DHCP lease DNS server', default = SETTINGS['DHCP_DNS']) parser.add_argument('-c', '--dhcp-broadcast', action = 'store', dest = 'DHCP_BROADCAST', help = 'DHCP broadcast address', default = SETTINGS['DHCP_BROADCAST']) parser.add_argument('-f', '--dhcp-fileserver', action = 'store', dest = 'DHCP_FILESERVER', help = 'DHCP fileserver IP', default = SETTINGS['DHCP_FILESERVER']) + parser.add_argument('--dhcp-whitelist', action = 'store_true', dest = 'DHCP_WHITELIST', help = 'Only respond to DHCP clients present in --static-config', default = False) # network boot directory and file name arguments parser.add_argument('-a', '--netboot-dir', action = 'store', dest = 'NETBOOT_DIR', help = 'Local file serve directory', default = SETTINGS['NETBOOT_DIR']) @@ -183,6 +184,7 @@ def parse_cli_arguments(): use_http = args.USE_HTTP, mode_proxy = args.DHCP_MODE_PROXY, mode_debug = ('dhcp' in args.MODE_DEBUG.lower() or 'all' in args.MODE_DEBUG.lower()), + whitelist = args.DHCP_WHITELIST, logger = dhcp_logger, static_config = loaded_statics) dhcpd = threading.Thread(target = dhcp_server.listen) diff --git a/pypxe/dhcp.py b/pypxe/dhcp.py index 358d58f..9bc3745 100644 --- a/pypxe/dhcp.py +++ b/pypxe/dhcp.py @@ -43,6 +43,7 @@ def __init__(self, **server_settings): self.mode_proxy = server_settings.get('mode_proxy', False) # ProxyDHCP mode self.mode_debug = server_settings.get('mode_debug', False) # debug mode self.static_config = server_settings.get('static_config', dict()) + self.whitelist = server_settings.get('whitelist', False) self.magic = struct.pack('!I', 0x63825363) # magic cookie self.logger = server_settings.get('logger', None) @@ -277,6 +278,9 @@ def dhcp_ack(self, message): def validate_req(self, client_mac): # client request is valid only if contains Vendor-Class = PXEClient + if self.whitelist and self.print_mac(client_mac) not in self.getNamespacedStatic('dhcp.binding'): + self.logger.debug('Non-whitelisted client request received') + return False if 60 in self.leases[client_mac]["options"] and 'PXEClient' in self.leases[client_mac]["options"][60][0]: self.logger.debug('PXE client request received') return True @@ -288,7 +292,7 @@ def listen(self): '''Main listen loop.''' while True: message, address = self.sock.recvfrom(1024) - client_mac = struct.unpack('!28x6s', message[:34]) + [client_mac] = struct.unpack('!28x6s', message[:34]) self.logger.debug('Received message') self.logger.debug('<--BEGIN MESSAGE-->') self.logger.debug('{0}'.format(repr(message))) From 4be5102c17de9086e8c7314cc49b1f579b76a62e Mon Sep 17 00:00:00 2001 From: PsychoMario Date: Mon, 11 May 2015 00:19:45 +0100 Subject: [PATCH 05/39] Added static-config and whitelist to README --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index ff1fd23..ab91e2b 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,7 @@ The following are arguments that can be passed to `pypxe-server.py` when running |__`--no-tftp`__|Disable built-in TFTP server which is enabled by default|`False`| |__`--debug`__|Enable selected services in DEBUG mode; services are selected by passing the name in a comma separated list. **Options are: http, tftp and dhcp** _This adds a level of verbosity so that you can see what's happening in the background._|`''`| |__`--config`__|Load configuration from JSON file. (see [`example_cfg.json`](example_cfg.json))|`None`| +|__`--static-config`__|Load DHCP lease configuration from JSON file. (see [`example-leases.json`](example-leases.json))|`None`| |__`--syslog`__|Specify a syslog server|`None`| |__`--syslog-port`__|Specify a syslog server port|`514`| @@ -66,6 +67,7 @@ The following are arguments that can be passed to `pypxe-server.py` when running |__`-d DHCP_DNS`__ or __`--dhcp-dns DHCP_DNS`__|Specify DHCP lease DNS server|`8.8.8.8`| |__`-c DHCP_BROADCAST`__ or __`--dhcp-broadcast DHCP_BROADCAST`__|Specify DHCP broadcast address|`''`| |__`-f DHCP_FILESERVER_IP`__ or __`--dhcp-fileserver-ip DHCP_FILESERVER_IP`__|Specify DHCP file server IP address|`192.168.2.2`| +|__`--dhcp-whitelist`__|Only serve clients specified in the static lease file (`--static-config`)|`False`| ##### File Name/Directory Arguments From c91883633308c800dabe0badc237312de7a5ed2e Mon Sep 17 00:00:00 2001 From: PsychoMario Date: Mon, 11 May 2015 00:31:37 +0100 Subject: [PATCH 06/39] fixed name changes bug, added logging --- example-leases.json | 6 +++--- pypxe/dhcp.py | 22 ++++++++++++++-------- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/example-leases.json b/example-leases.json index 7b82aa2..c07150c 100644 --- a/example-leases.json +++ b/example-leases.json @@ -1,13 +1,13 @@ { "dhcp": { "binding": { - "00:26:B9:A0:97:29": { + "5C:9A:D8:5E:F2:E5": { "dns": [ "8.8.8.8", "8.8.4.4" ], - "ipaddr": "192.168.2.123", - "router": "192.168.2.5", + "ipaddr": "192.168.0.123", + "router": "192.168.0.1", "subnet": "255.255.255.0" } } diff --git a/pypxe/dhcp.py b/pypxe/dhcp.py index 9bc3745..691833e 100644 --- a/pypxe/dhcp.py +++ b/pypxe/dhcp.py @@ -76,12 +76,18 @@ def __init__(self, **server_settings): self.logger.debug('DHCP DNS Server: {0}'.format(self.dns_server)) self.logger.debug('DHCP Broadcast Address: {0}'.format(self.broadcast)) + if self.static_config: + self.logger.debug('DHCP Using Static Leasing') + self.logger.debug('DHCP Using Static Leasing Whitelist: {0}'.format(self.whitelist)) + self.logger.debug('DHCP File Server IP: {0}'.format(self.file_server)) self.logger.debug('DHCP File Name: {0}'.format(self.file_name)) self.logger.debug('ProxyDHCP Mode: {0}'.format(self.mode_proxy)) self.logger.debug('Using iPXE: {0}'.format(self.ipxe)) self.logger.debug('Using HTTP Server: {0}'.format(self.http)) + + self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) @@ -172,7 +178,7 @@ def craft_header(self, message): if self.leases[client_mac]['ip']: # OFFER offer = self.leases[client_mac]['ip'] else: # ACK - offer = self.getNamespacedStatic('dhcp.binding.{}.ipaddr'.format(self.printMAC(clientmac))) + offer = self.getNamespacedStatic('dhcp.binding.{}.ipaddr'.format(self.print_mac(client_mac))) offer = offer if offer else self.next_ip() self.leases[client_mac]['ip'] = offer self.leases[client_mac]['expire'] = time() + 86400 @@ -205,14 +211,14 @@ def craft_options(self, opt53, client_mac): response = self.tlv_encode(53, chr(opt53)) # message type, OFFER response += self.tlv_encode(54, socket.inet_aton(self.ip)) # DHCP Server if not self.mode_proxy: - subnetmask = self.getNamespacedStatic('dhcp.binding.{}.subnet'.format(self.printMAC(clientmac)), self.subnetmask) - response += self.tlvEncode(1, socket.inet_aton(subnetmask)) #SubnetMask - router = self.getNamespacedStatic('dhcp.binding.{}.router'.format(self.printMAC(clientmac)), self.router) - response += self.tlvEncode(3, socket.inet_aton(router)) #Router - dnsserver = self.getNamespacedStatic('dhcp.binding.{}.dns'.format(self.printMAC(clientmac)), [self.dnsserver]) + subnetmask = self.getNamespacedStatic('dhcp.binding.{}.subnet'.format(self.print_mac(client_mac)), self.subnet_mask) + response += self.tlv_encode(1, socket.inet_aton(subnetmask)) #SubnetMask + router = self.getNamespacedStatic('dhcp.binding.{}.router'.format(self.print_mac(client_mac)), self.router) + response += self.tlv_encode(3, socket.inet_aton(router)) #Router + dnsserver = self.getNamespacedStatic('dhcp.binding.{}.dns'.format(self.print_mac(client_mac)), [self.dns_server]) dnsserver = ''.join([socket.inet_aton(i) for i in dnsserver]) - response += self.tlvEncode(6, dnsserver) - response += self.tlvEncode(51, struct.pack('!I', 86400)) #lease time + response += self.tlv_encode(6, dnsserver) + response += self.tlv_encode(51, struct.pack('!I', 86400)) #lease time # TFTP Server OR HTTP Server; if iPXE, need both response += self.tlv_encode(66, self.file_server) From b1a30e5cff7fc7cd81e9539fb9121075c4bf07ee Mon Sep 17 00:00:00 2001 From: PsychoMario Date: Fri, 17 Apr 2015 02:50:30 +0100 Subject: [PATCH 07/39] First NBD commit. Readonly functional --- pypxe/nbd/nbd.py | 119 ++++++++++++++++++++++++++++++++++++++++++++++ pypxe/nbd/test.py | 9 ++++ 2 files changed, 128 insertions(+) create mode 100644 pypxe/nbd/nbd.py create mode 100644 pypxe/nbd/test.py diff --git a/pypxe/nbd/nbd.py b/pypxe/nbd/nbd.py new file mode 100644 index 0000000..896851c --- /dev/null +++ b/pypxe/nbd/nbd.py @@ -0,0 +1,119 @@ +import logging +import socket +import struct + +class NBD: + def __init__(self, **serverSettings): + self.bd = serverSettings.get('blockdevice', '') + self.mode = serverSettings.get('mode', 'r') #r/w + self.ip = serverSettings.get('ip', '0.0.0.0') + self.port = serverSettings.get('port', 10809) + self.mode_debug = serverSettings.get('mode_debug', False) #debug mode + self.logger = serverSettings.get('logger', None) + + # setup logger + if self.logger == None: + self.logger = logging.getLogger("NBD") + handler = logging.StreamHandler() + formatter = logging.Formatter('%(asctime)s %(name)s [%(levelname)s] %(message)s') + handler.setFormatter(formatter) + self.logger.addHandler(handler) + + if self.mode_debug: + self.logger.setLevel(logging.DEBUG) + + self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + self.sock.bind((self.ip, self.port)) + self.sock.listen(1) + + self.openbd = open(self.bd, 'ra' if self.mode == 'w' else self.mode) + # go to EOF + self.openbd.seek(0, 2) + self.bdsize = self.openbd.tell() + # go back to start + self.openbd.seek(0) + + self.logger.debug('NOTICE: NBD server started in debug mode. NBD server is using the following:') + self.logger.debug(' NBD Server IP: {}'.format(self.ip)) + self.logger.debug(' NBD Server Port: {}'.format(self.port)) + self.logger.debug(' NBD Block Device: {}'.format(self.bd)) + + + def handshake(self, conn, addr): + handshake = 'NBDMAGIC' + conn.send(handshake) + # 0x49484156454F5054 + handshake = 'IHAVEOPT' + # fixed newstyle and nozeros + handshake += struct.pack("!H", 3) + conn.send(handshake) + + def exportMount(self, conn, addr): + [flags] = struct.unpack("!I", conn.recv(4)) + newstyle = flags & 1 + nozero = flags & 2 + # magic + conn.recv(8) + [option] = struct.unpack("!I", conn.recv(4)) + if option != 1: + #Isn't a NBD_OPT_EXPORT_NAME + conn.close() + return 1 + + [exportlen] = struct.unpack("!I", conn.recv(4)) + export = conn.recv(exportlen) + if export != self.bd: + conn.close() + return 1 + + self.logger.debug('Received request for %s from %s', export, addr) + + # size of export + exportinfo = struct.pack('!Q', self.bdsize) + flags = 1 # has flags + flags |= 1 << (self.mode == 'r') # readonly? + exportinfo += struct.pack('!H', flags) + exportinfo += "\x00"*(0 if nozero else 124) + conn.send(exportinfo) + + def handleClient(self, conn, addr): + self.handshake(conn, addr) + ret = self.exportMount(conn, addr) + if ret: return # client did something wrong, so we closed them + + while True: + # atrocious. but works. + while not conn.recv(4): pass + [opcode, handle, offset, length] = struct.unpack("!IQQI", conn.recv(24)) + if opcode not in (0, 1, 2): + # NBD_REP_ERR_UNSUP + response = struct.pack("!I", 0x67446698) + response += struct.pack("!I", opcode) + response += struct.pack("!I", 2**31 + 1) # NBD_REP_ERR_UNSUP + response += struct.pack("!I", 0) # length + conn.send(response) + if opcode == 0: # READ + self.openbd.seek(offset) + data = self.openbd.read(length) + response = struct.pack("!I", 0x67446698) + response += struct.pack("!I", 0) # error + response += struct.pack("!Q", handle) + response += data + conn.send(response) + self.logger.debug('%s read %d bytes from %s', addr, length, hex(offset)) + elif opcode == 1: # WRITE + data = conn.recv(length) + self.logger.debug('%s wrote %d bytes from %s', addr, length, hex(offset)) + pass + elif opcode == 2: # DISCONNECT + conn.close() + self.logger.debug('%s disconnected', addr) + return + + def listen(self): + '''This method is the main loop that listens for requests''' + while True: + conn, addr = self.sock.accept() + # should probably fork these + self.handleClient(conn, addr) diff --git a/pypxe/nbd/test.py b/pypxe/nbd/test.py new file mode 100644 index 0000000..1100203 --- /dev/null +++ b/pypxe/nbd/test.py @@ -0,0 +1,9 @@ +import nbd + +args = { + 'blockdevice':'blank.img', + 'mode_debug':True, + 'mode':'w' + } + +nbd.NBD(**args).listen() From 35f3e1588e3a68ba6c34c57207908414daef6f16 Mon Sep 17 00:00:00 2001 From: PsychoMario Date: Fri, 17 Apr 2015 18:42:17 +0100 Subject: [PATCH 08/39] rewrote handshake, readonly hangs --- pypxe/nbd/nbd.py | 68 +++++++++++++++++++++++------------------------ pypxe/nbd/test.py | 2 +- 2 files changed, 34 insertions(+), 36 deletions(-) diff --git a/pypxe/nbd/nbd.py b/pypxe/nbd/nbd.py index 896851c..a3bdbc2 100644 --- a/pypxe/nbd/nbd.py +++ b/pypxe/nbd/nbd.py @@ -39,72 +39,70 @@ def __init__(self, **serverSettings): self.logger.debug(' NBD Server Port: {}'.format(self.port)) self.logger.debug(' NBD Block Device: {}'.format(self.bd)) + def sendreply(self, conn, addr, code, data): + reply = struct.pack("!Q", 0x3e889045565a9) + reply += struct.pack("!I", code) + reply += struct.pack("!I", len(data)) + reply += data + conn.send(reply) def handshake(self, conn, addr): - handshake = 'NBDMAGIC' - conn.send(handshake) + # Mostly taken from https://github.com/yoe/nbd/blob/master/nbd-server.c + conn.send('NBDMAGIC') # 0x49484156454F5054 - handshake = 'IHAVEOPT' - # fixed newstyle and nozeros - handshake += struct.pack("!H", 3) - conn.send(handshake) - - def exportMount(self, conn, addr): - [flags] = struct.unpack("!I", conn.recv(4)) - newstyle = flags & 1 - nozero = flags & 2 - # magic - conn.recv(8) - [option] = struct.unpack("!I", conn.recv(4)) - if option != 1: - #Isn't a NBD_OPT_EXPORT_NAME - conn.close() - return 1 + conn.send('IHAVEOPT') + # NBD_FLAG_FIXED_NEWSTYLE | NBD_FLAG_NO_ZEROES + conn.send(struct.pack("!H", 3)) - [exportlen] = struct.unpack("!I", conn.recv(4)) - export = conn.recv(exportlen) - if export != self.bd: + [cflags] = struct.unpack("!I", conn.recv(4)) + + op = 0 + while op != 1: # NBD_OPT_EXPORT_NAME + [magic] = struct.unpack("!Q", conn.recv(8)) + [op] = struct.unpack("!I", conn.recv(4)) + if op != 1: + # NBD_REP_ERR_UNSUP + self.sendreply(conn, addr, 2**31+1, '') + + [namelen] = struct.unpack("!I", conn.recv(4)) + name = conn.recv(namelen) + if name != self.bd: conn.close() return 1 - self.logger.debug('Received request for %s from %s', export, addr) + self.logger.debug('Received request for %s from %s', name, addr) # size of export exportinfo = struct.pack('!Q', self.bdsize) - flags = 1 # has flags - flags |= 1 << (self.mode == 'r') # readonly? + flags = int((self.mode == 'r')) # readonly? exportinfo += struct.pack('!H', flags) - exportinfo += "\x00"*(0 if nozero else 124) + exportinfo += "\x00"*(0 if (cflags&2) else 124) conn.send(exportinfo) def handleClient(self, conn, addr): - self.handshake(conn, addr) - ret = self.exportMount(conn, addr) + ret = self.handshake(conn, addr) if ret: return # client did something wrong, so we closed them while True: - # atrocious. but works. + # atrocious. but works. MSG_WAITALL seems non-functional while not conn.recv(4): pass [opcode, handle, offset, length] = struct.unpack("!IQQI", conn.recv(24)) if opcode not in (0, 1, 2): # NBD_REP_ERR_UNSUP - response = struct.pack("!I", 0x67446698) - response += struct.pack("!I", opcode) - response += struct.pack("!I", 2**31 + 1) # NBD_REP_ERR_UNSUP - response += struct.pack("!I", 0) # length - conn.send(response) + self.sendreply(conn, addr, 2**31+1, '') + continue if opcode == 0: # READ self.openbd.seek(offset) data = self.openbd.read(length) response = struct.pack("!I", 0x67446698) response += struct.pack("!I", 0) # error response += struct.pack("!Q", handle) - response += data conn.send(response) + conn.send(data) self.logger.debug('%s read %d bytes from %s', addr, length, hex(offset)) elif opcode == 1: # WRITE data = conn.recv(length) - self.logger.debug('%s wrote %d bytes from %s', addr, length, hex(offset)) + self.logger.debug('%s wrote %d bytes to %s', addr, length, hex(offset)) pass elif opcode == 2: # DISCONNECT conn.close() diff --git a/pypxe/nbd/test.py b/pypxe/nbd/test.py index 1100203..801f6b7 100644 --- a/pypxe/nbd/test.py +++ b/pypxe/nbd/test.py @@ -3,7 +3,7 @@ args = { 'blockdevice':'blank.img', 'mode_debug':True, - 'mode':'w' + 'mode':'r' } nbd.NBD(**args).listen() From f7edfaa6bb156b41f0b4869e20ce0ab2ca2513a4 Mon Sep 17 00:00:00 2001 From: PsychoMario Date: Sat, 18 Apr 2015 00:36:45 +0100 Subject: [PATCH 09/39] fixed readonly flag causing hang --- pypxe/nbd/nbd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pypxe/nbd/nbd.py b/pypxe/nbd/nbd.py index a3bdbc2..cab2007 100644 --- a/pypxe/nbd/nbd.py +++ b/pypxe/nbd/nbd.py @@ -74,7 +74,7 @@ def handshake(self, conn, addr): # size of export exportinfo = struct.pack('!Q', self.bdsize) - flags = int((self.mode == 'r')) # readonly? + flags = (2 if self.mode == 'r' else 0) # readonly? exportinfo += struct.pack('!H', flags) exportinfo += "\x00"*(0 if (cflags&2) else 124) conn.send(exportinfo) From 96f02ce5fc4452ac44bc6c854249939f11fa3102 Mon Sep 17 00:00:00 2001 From: PsychoMario Date: Sat, 18 Apr 2015 01:06:17 +0100 Subject: [PATCH 10/39] fixed open modes, added write support --- pypxe/nbd/nbd.py | 12 +++++++++++- pypxe/nbd/test.py | 2 +- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/pypxe/nbd/nbd.py b/pypxe/nbd/nbd.py index cab2007..4f8e24c 100644 --- a/pypxe/nbd/nbd.py +++ b/pypxe/nbd/nbd.py @@ -27,9 +27,10 @@ def __init__(self, **serverSettings): self.sock.bind((self.ip, self.port)) self.sock.listen(1) - self.openbd = open(self.bd, 'ra' if self.mode == 'w' else self.mode) + self.openbd = open(self.bd, 'r+b' if self.mode == 'rw' else 'rb') # go to EOF self.openbd.seek(0, 2) + # we need this when clients mount us self.bdsize = self.openbd.tell() # go back to start self.openbd.seek(0) @@ -38,6 +39,7 @@ def __init__(self, **serverSettings): self.logger.debug(' NBD Server IP: {}'.format(self.ip)) self.logger.debug(' NBD Server Port: {}'.format(self.port)) self.logger.debug(' NBD Block Device: {}'.format(self.bd)) + self.logger.debug(' NBD Block Device Mode: {}'.format(self.mode)) def sendreply(self, conn, addr, code, data): reply = struct.pack("!Q", 0x3e889045565a9) @@ -101,7 +103,15 @@ def handleClient(self, conn, addr): conn.send(data) self.logger.debug('%s read %d bytes from %s', addr, length, hex(offset)) elif opcode == 1: # WRITE + # don't think we need to check RO + # COW goes here data = conn.recv(length) + self.openbd.seek(offset) + self.openbd.write(data) + response = struct.pack("!I", 0x67446698) + response += struct.pack("!I", 0) # error + response += struct.pack("!Q", handle) + conn.send(response) self.logger.debug('%s wrote %d bytes to %s', addr, length, hex(offset)) pass elif opcode == 2: # DISCONNECT diff --git a/pypxe/nbd/test.py b/pypxe/nbd/test.py index 801f6b7..2998b05 100644 --- a/pypxe/nbd/test.py +++ b/pypxe/nbd/test.py @@ -3,7 +3,7 @@ args = { 'blockdevice':'blank.img', 'mode_debug':True, - 'mode':'r' + 'mode':'rw' } nbd.NBD(**args).listen() From b6505c711737e778578f0243f65b0ef2c371089c Mon Sep 17 00:00:00 2001 From: PsychoMario Date: Sat, 18 Apr 2015 15:05:48 +0100 Subject: [PATCH 11/39] added bootinfo to test.py --- pypxe/nbd/test.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pypxe/nbd/test.py b/pypxe/nbd/test.py index 2998b05..413220f 100644 --- a/pypxe/nbd/test.py +++ b/pypxe/nbd/test.py @@ -1,5 +1,8 @@ import nbd - +# kernel cmdline options: +# https://wiki.archlinux.org/index.php/Syslinux#Pxelinux +# initrd build config: +# https://wiki.archlinux.org/index.php/Diskless_system#Bootstrapping_installation args = { 'blockdevice':'blank.img', 'mode_debug':True, From 643419c08aa7ed58e234a7947064a5e288d2a090 Mon Sep 17 00:00:00 2001 From: PsychoMario Date: Sun, 19 Apr 2015 00:20:05 +0100 Subject: [PATCH 12/39] threading, cow --- pypxe/nbd/nbd.py | 70 ++++++++++++++++++-------- pypxe/nbd/test.py | 8 ++- pypxe/nbd/writes.py | 120 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 174 insertions(+), 24 deletions(-) create mode 100644 pypxe/nbd/writes.py diff --git a/pypxe/nbd/nbd.py b/pypxe/nbd/nbd.py index 4f8e24c..c8f3e6a 100644 --- a/pypxe/nbd/nbd.py +++ b/pypxe/nbd/nbd.py @@ -1,11 +1,16 @@ import logging import socket import struct +import threading +import sys +import os +import writes class NBD: def __init__(self, **serverSettings): self.bd = serverSettings.get('blockdevice', '') - self.mode = serverSettings.get('mode', 'r') #r/w + self.write = serverSettings.get('write', False) #w? + self.cow = serverSettings.get('cow', True) # COW is the safe default self.ip = serverSettings.get('ip', '0.0.0.0') self.port = serverSettings.get('port', 10809) self.mode_debug = serverSettings.get('mode_debug', False) #debug mode @@ -25,9 +30,10 @@ def __init__(self, **serverSettings): self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self.sock.bind((self.ip, self.port)) - self.sock.listen(1) + self.sock.listen(4) - self.openbd = open(self.bd, 'r+b' if self.mode == 'rw' else 'rb') + # if we have COW on, we write elsewhere so we don't need write ability + self.openbd = open(self.bd, 'r+b' if self.write and not self.cow else 'rb') # go to EOF self.openbd.seek(0, 2) # we need this when clients mount us @@ -39,9 +45,11 @@ def __init__(self, **serverSettings): self.logger.debug(' NBD Server IP: {}'.format(self.ip)) self.logger.debug(' NBD Server Port: {}'.format(self.port)) self.logger.debug(' NBD Block Device: {}'.format(self.bd)) - self.logger.debug(' NBD Block Device Mode: {}'.format(self.mode)) + self.logger.debug(' NBD Block Device Writes: {}'.format(self.write)) + self.logger.debug(' NBD Block Write Method: {}'.format("Copy-On-Write" if self.cow else "File")) def sendreply(self, conn, addr, code, data): + '''Send a reply with magic. only used for error codes''' reply = struct.pack("!Q", 0x3e889045565a9) reply += struct.pack("!I", code) reply += struct.pack("!I", len(data)) @@ -49,6 +57,7 @@ def sendreply(self, conn, addr, code, data): conn.send(reply) def handshake(self, conn, addr): + '''Initiate the connection. Server sends first.''' # Mostly taken from https://github.com/yoe/nbd/blob/master/nbd-server.c conn.send('NBDMAGIC') # 0x49484156454F5054 @@ -76,52 +85,69 @@ def handshake(self, conn, addr): # size of export exportinfo = struct.pack('!Q', self.bdsize) - flags = (2 if self.mode == 'r' else 0) # readonly? + flags = (0 if self.write else 2) # readonly? exportinfo += struct.pack('!H', flags) exportinfo += "\x00"*(0 if (cflags&2) else 124) conn.send(exportinfo) - def handleClient(self, conn, addr): + def handleClient(self, conn, addr, seeklock): + '''Handle all client actions, R/W/Disconnect''' ret = self.handshake(conn, addr) if ret: return # client did something wrong, so we closed them + FS = writes.write(self.cow)(addr, self.openbd, self.logger, seeklock) + while True: - # atrocious. but works. MSG_WAITALL seems non-functional - while not conn.recv(4): pass - [opcode, handle, offset, length] = struct.unpack("!IQQI", conn.recv(24)) + conn.recv(4) + [opcode, handle, offset, length] = struct.unpack("!IQQI", conn.recv(24, socket.MSG_WAITALL)) if opcode not in (0, 1, 2): # NBD_REP_ERR_UNSUP self.sendreply(conn, addr, 2**31+1, '') continue if opcode == 0: # READ - self.openbd.seek(offset) - data = self.openbd.read(length) + data = FS.read(offset, length) + response = struct.pack("!I", 0x67446698) response += struct.pack("!I", 0) # error response += struct.pack("!Q", handle) conn.send(response) conn.send(data) - self.logger.debug('%s read %d bytes from %s', addr, length, hex(offset)) + elif opcode == 1: # WRITE - # don't think we need to check RO - # COW goes here - data = conn.recv(length) - self.openbd.seek(offset) - self.openbd.write(data) + # WAIT because if there's any lag at all we don't get the whole + # thing, we don't write the whole thing, and then we break + # trying to parse the rest of the data + data = conn.recv(length, socket.MSG_WAITALL) + FS.write(offset, data) + response = struct.pack("!I", 0x67446698) response += struct.pack("!I", 0) # error response += struct.pack("!Q", handle) conn.send(response) - self.logger.debug('%s wrote %d bytes to %s', addr, length, hex(offset)) - pass + elif opcode == 2: # DISCONNECT + # delete COW diff conn.close() self.logger.debug('%s disconnected', addr) return def listen(self): '''This method is the main loop that listens for requests''' + seeklock = threading.Lock() + cowfiles = [] while True: - conn, addr = self.sock.accept() - # should probably fork these - self.handleClient(conn, addr) + try: + conn, addr = self.sock.accept() + # Split off on a thread. Allows us to handle multiple clients + dispatch = threading.Thread(target = self.handleClient, args = (conn, addr, seeklock)) + # clients don't necessarily close the TCP connection + # so we use this to kill the program on Ctrl-c + dispatch.daemon = True + dispatch.start() + # this is for the cleanup at the end. Will need clarifying + # if MemCOW + if self.cow: + cowfiles.append('PyPXE_NBD_COW_%s_%s' % addr) + except KeyboardInterrupt: + map(os.remove, cowfiles) + return diff --git a/pypxe/nbd/test.py b/pypxe/nbd/test.py index 413220f..8b790e4 100644 --- a/pypxe/nbd/test.py +++ b/pypxe/nbd/test.py @@ -4,9 +4,13 @@ # initrd build config: # https://wiki.archlinux.org/index.php/Diskless_system#Bootstrapping_installation args = { - 'blockdevice':'blank.img', + 'blockdevice':'arch.img', 'mode_debug':True, - 'mode':'rw' + 'write':True, + 'cow':True } +# rw and multiple clients not safe +# rcow/ro and multiple clients OK +# need a 'mode' and a 'cow' arg nbd.NBD(**args).listen() diff --git a/pypxe/nbd/writes.py b/pypxe/nbd/writes.py new file mode 100644 index 0000000..db68537 --- /dev/null +++ b/pypxe/nbd/writes.py @@ -0,0 +1,120 @@ +class DiskCOW: + # Named to allow MemCOW at a later date + def __init__(self, addr, imagefd, logger, seeklock): + # Optional argset for: + # disk diff path + self.addr = addr + self.imagefd = imagefd + self.seeklock = seeklock + self.logger = logger.getChild('FS') + self.logger.debug('Copy-On-Write for %s in PyPXE_NBD_COW_%s_%s', addr, *addr) + + #never want readonly cow, also definately creating file + self.fh = open('PyPXE_NBD_COW_%s_%s' % addr, 'w+b') + # pages is a list of the addresses for which we have different pages + # should all be multiples of 4096. + self.pages = [] + + def basepages(self, offset, length): + # basepages is (page base addr, inter page offset, length of data in page) + # it's unlikely we'll ever need sub 4096 reads, but just in case. + # I have observed sub 4096 writes so that justifies it. + basepages = [] + + # first chunk, not necessarily at page boundary + basepages.append((offset - (offset % 4096), offset % 4096, 4096 - (offset % 4096))) + length -= 4096 - (offset % 4096) + offset += 4096 + + # all following FULL chunks. definate page boundary and full size + while length >= 4096: + basepages.append((offset, 0, 4096)) + length -= 4096 + offset += 4096 + + # final non-full chunk, definate offset, variable length + if length > 0: + basepages.append((offset, 0, length)) + + return basepages + + def read(self, offset, length): + basepages = self.basepages(offset, length) + + self.logger.debug('%s reading %d bytes from %s. Pages: %s', self.addr, length, hex(offset), len(basepages)) + + data = "" + for major, minor, length in basepages: + if major in self.pages: + # major is the nth page in the file + off = self.pages.index(major) + self.fh.seek(off*4096 + minor) + data += self.fh.read(length) + else: + # This is a race condition. If another thread seeks after we've + # seeked, but before we read, the seek is changed and the data + # is wrong. + # Lock is shared between all clients. Only applies to imagefd, + # self.fd is unique and per client. + self.seeklock.acquire() + self.imagefd.seek(major + minor) + data += self.imagefd.read(length) + self.seeklock.release() + return data + + def write(self, offset, data): + basepages = self.basepages(offset, len(data)) + + self.logger.debug('%s writing %d bytes to %s. Pages: %s', self.addr, len(data), hex(offset), len(basepages)) + + for major, minor, length in basepages: + if major in self.pages: + # we already have a copied page, so we can just overwrite it + self.fh.seek(major + minor) + self.fh.write(data[:length]) + data = data[length:] + else: + # we don't have this page, so copy it first. then add to the list + self.seeklock.acquire() + # on the page boundary + self.imagefd.seek(major) + cpdata = self.imagefd.read(4096) + self.seeklock.release() + # append to EOF + self.fh.seek(0, 2) + self.fh.write(cpdata) + self.pages.append(major) + # we've got a copy of the page now, we just need to write it + off = self.pages.index(major) + self.fh.seek(off*4096 + minor) + self.fh.write(data[:length]) + data = data[length:] + +class RW: + def __init__(self, addr, imagefd, logger, seeklock): + self.addr = addr + self.seeklock = seeklock + self.imagefd = imagefd + self.logger = logger.getChild('FS') + self.logger.debug('File for %s', addr) + + def read(self, offset, length): + self.logger.debug('%s reading %d bytes from %s', self.addr, length, hex(offset)) + # see COW.read() for lock reason + self.seeklock.acquire() + self.imagefd.seek(offset) + data = self.imagefd.read(length) + self.seeklock.release() + return data + + def write(self, offset, data): + self.logger.debug('%s writing %d bytes to %s', self.addr, len(data), hex(offset)) + self.seeklock.acquire() + self.imagefd.seek(offset) + self.imagefd.write(data) + self.seeklock.release() + +def write(cow): + '''Class signatures are identical so we can transparently + use either.''' + return DiskCOW if cow else RW From ba5447aa939e3da2ca4af210202ab9805f35fa08 Mon Sep 17 00:00:00 2001 From: PsychoMario Date: Sun, 19 Apr 2015 01:40:44 +0100 Subject: [PATCH 13/39] Added MemCOW, copytoram --- pypxe/nbd/nbd.py | 24 ++++++++++++------- pypxe/nbd/test.py | 9 +++---- pypxe/nbd/writes.py | 57 +++++++++++++++++++++++++++++++-------------- 3 files changed, 58 insertions(+), 32 deletions(-) diff --git a/pypxe/nbd/nbd.py b/pypxe/nbd/nbd.py index c8f3e6a..7c5222b 100644 --- a/pypxe/nbd/nbd.py +++ b/pypxe/nbd/nbd.py @@ -4,6 +4,7 @@ import threading import sys import os +import io import writes class NBD: @@ -11,6 +12,8 @@ def __init__(self, **serverSettings): self.bd = serverSettings.get('blockdevice', '') self.write = serverSettings.get('write', False) #w? self.cow = serverSettings.get('cow', True) # COW is the safe default + self.inmem = serverSettings.get('inmem', False) + self.copytoram = serverSettings.get('copytoram', False) self.ip = serverSettings.get('ip', '0.0.0.0') self.port = serverSettings.get('port', 10809) self.mode_debug = serverSettings.get('mode_debug', False) #debug mode @@ -24,6 +27,13 @@ def __init__(self, **serverSettings): handler.setFormatter(formatter) self.logger.addHandler(handler) + self.logger.debug('NOTICE: NBD server started in debug mode. NBD server is using the following:') + self.logger.debug(' NBD Server IP: {}'.format(self.ip)) + self.logger.debug(' NBD Server Port: {}'.format(self.port)) + self.logger.debug(' NBD Block Device: {}'.format(self.bd)) + self.logger.debug(' NBD Block Device Writes: {}'.format(self.write)) + self.logger.debug(' NBD Block Write Method: {} ({})'.format("Copy-On-Write" if self.cow else "File", "Memory" if self.inmem else "Disk")) + if self.mode_debug: self.logger.setLevel(logging.DEBUG) @@ -40,13 +50,11 @@ def __init__(self, **serverSettings): self.bdsize = self.openbd.tell() # go back to start self.openbd.seek(0) + if self.copytoram and self.cow: + self.logger.info('Starting copying %s to RAM', self.bd) + self.openbd = io.BytesIO(self.openbd.read()) + self.logger.info('Finished copying %s to RAM', self.bd) - self.logger.debug('NOTICE: NBD server started in debug mode. NBD server is using the following:') - self.logger.debug(' NBD Server IP: {}'.format(self.ip)) - self.logger.debug(' NBD Server Port: {}'.format(self.port)) - self.logger.debug(' NBD Block Device: {}'.format(self.bd)) - self.logger.debug(' NBD Block Device Writes: {}'.format(self.write)) - self.logger.debug(' NBD Block Write Method: {}'.format("Copy-On-Write" if self.cow else "File")) def sendreply(self, conn, addr, code, data): '''Send a reply with magic. only used for error codes''' @@ -95,7 +103,7 @@ def handleClient(self, conn, addr, seeklock): ret = self.handshake(conn, addr) if ret: return # client did something wrong, so we closed them - FS = writes.write(self.cow)(addr, self.openbd, self.logger, seeklock) + FS = writes.write(self.cow, self.inmem)(addr, self.openbd, self.logger, seeklock) while True: conn.recv(4) @@ -146,7 +154,7 @@ def listen(self): dispatch.start() # this is for the cleanup at the end. Will need clarifying # if MemCOW - if self.cow: + if self.cow and not self.inmem: cowfiles.append('PyPXE_NBD_COW_%s_%s' % addr) except KeyboardInterrupt: map(os.remove, cowfiles) diff --git a/pypxe/nbd/test.py b/pypxe/nbd/test.py index 8b790e4..0478f2a 100644 --- a/pypxe/nbd/test.py +++ b/pypxe/nbd/test.py @@ -1,16 +1,13 @@ import nbd -# kernel cmdline options: -# https://wiki.archlinux.org/index.php/Syslinux#Pxelinux -# initrd build config: -# https://wiki.archlinux.org/index.php/Diskless_system#Bootstrapping_installation args = { 'blockdevice':'arch.img', 'mode_debug':True, 'write':True, - 'cow':True + 'cow':True, + 'inmem':True, + 'copytoram':True } # rw and multiple clients not safe # rcow/ro and multiple clients OK -# need a 'mode' and a 'cow' arg nbd.NBD(**args).listen() diff --git a/pypxe/nbd/writes.py b/pypxe/nbd/writes.py index db68537..616099b 100644 --- a/pypxe/nbd/writes.py +++ b/pypxe/nbd/writes.py @@ -1,20 +1,6 @@ -class DiskCOW: - # Named to allow MemCOW at a later date - def __init__(self, addr, imagefd, logger, seeklock): - # Optional argset for: - # disk diff path - self.addr = addr - self.imagefd = imagefd - self.seeklock = seeklock - self.logger = logger.getChild('FS') - self.logger.debug('Copy-On-Write for %s in PyPXE_NBD_COW_%s_%s', addr, *addr) - - #never want readonly cow, also definately creating file - self.fh = open('PyPXE_NBD_COW_%s_%s' % addr, 'w+b') - # pages is a list of the addresses for which we have different pages - # should all be multiples of 4096. - self.pages = [] +import io +class COW: def basepages(self, offset, length): # basepages is (page base addr, inter page offset, length of data in page) # it's unlikely we'll ever need sub 4096 reads, but just in case. @@ -90,6 +76,36 @@ def write(self, offset, data): self.fh.write(data[:length]) data = data[length:] +class DiskCOW(COW): + def __init__(self, addr, imagefd, logger, seeklock): + # Optional argset for: + # disk diff path + self.addr = addr + self.imagefd = imagefd + self.seeklock = seeklock + self.logger = logger.getChild('FS') + self.logger.debug('Copy-On-Write for %s in PyPXE_NBD_COW_%s_%s', addr, *addr) + + #never want readonly cow, also definately creating file + self.fh = open('PyPXE_NBD_COW_%s_%s' % addr, 'w+b') + # pages is a list of the addresses for which we have different pages + # should all be multiples of 4096. + self.pages = [] + +class MemCOW(COW): + def __init__(self, addr, imagefd, logger, seeklock): + self.addr = addr + self.imagefd = imagefd + self.seeklock = seeklock + self.logger = logger.getChild('FS') + self.logger.debug('Copy-On-Write for %s in Memory', addr) + + #BytesIO looks exactly the same as a file, perfect for in memory disk + self.fh = io.BytesIO() + # pages is a list of the addresses for which we have different pages + # should all be multiples of 4096. + self.pages = [] + class RW: def __init__(self, addr, imagefd, logger, seeklock): self.addr = addr @@ -114,7 +130,12 @@ def write(self, offset, data): self.imagefd.write(data) self.seeklock.release() -def write(cow): +def write(cow, inmem): '''Class signatures are identical so we can transparently use either.''' - return DiskCOW if cow else RW + if cow and inmem: + return MemCOW + elif cow and not inmem: + return DiskCOW + else: + return RW From a271499de6ffab82f80e5a2d6b49fb9fdb5d91c9 Mon Sep 17 00:00:00 2001 From: PsychoMario Date: Sun, 19 Apr 2015 17:43:01 +0100 Subject: [PATCH 14/39] added arguments for running and integrated, caught malformed packet error --- netboot/boot.http.nbd.ipxe | 4 +++ pypxe-server.py | 73 +++++++++++++++++++++++++++++++------- pypxe/nbd/__init__.py | 1 + pypxe/nbd/nbd.py | 9 ++++- pypxe/nbd/writes.py | 4 +++ 5 files changed, 78 insertions(+), 13 deletions(-) create mode 100644 netboot/boot.http.nbd.ipxe create mode 100644 pypxe/nbd/__init__.py diff --git a/netboot/boot.http.nbd.ipxe b/netboot/boot.http.nbd.ipxe new file mode 100644 index 0000000..16ef087 --- /dev/null +++ b/netboot/boot.http.nbd.ipxe @@ -0,0 +1,4 @@ +#!ipxe +initrd http://192.168.0.11/initramfs-linux.img +chain http://192.168.0.11/vmlinuz-linux quiet ip=::::::dhcp nbd_host=192.168.0.11 nbd_name=arch.img root=/dev/nbd0 +sanboot diff --git a/pypxe-server.py b/pypxe-server.py index 5d92355..db49061 100755 --- a/pypxe-server.py +++ b/pypxe-server.py @@ -35,6 +35,13 @@ 'USE_TFTP':True, 'USE_DHCP':True, 'DHCP_MODE_PROXY':False, + 'NBD_BLOCKDEVICE':'', + 'NBD_WRITE':False, + 'NBD_COW':True, + 'NBD_COWINMEM':False, + 'NBD_COPYTORAM':False, + 'NBD_SERVER_IP':'0.0.0.0', + 'NBD_PORT':10809, 'MODE_DEBUG':''} def parse_cli_arguments(): @@ -49,20 +56,31 @@ def parse_cli_arguments(): parser.add_argument('--syslog', action = 'store', dest = 'SYSLOG_SERVER', help = 'Syslog server', default = SETTINGS['SYSLOG_SERVER']) parser.add_argument('--syslog-port', action = 'store', dest = 'SYSLOG_PORT', help = 'Syslog server port', default = SETTINGS['SYSLOG_PORT']) + # NBD server arguments + nbd_group = parser.add_argument_group(title = 'Network Block Device', description = 'Arguments relevant to the NBD server') + nbd_group.add_argument('--nbd', action = 'store', dest = 'NBD_BLOCKDEVICE', help = 'Enable the NDB server with a specific block device (Can be a disk image)', default = NBD_BLOCKDEVICE) + nbd_group.add_argument('--nbd-write', action = 'store_true', dest = 'NBD_WRITE', help = 'Enable writes on the NBD device', default = NBD_WRITE) + nbd_group.add_argument('--nbd-cow', action = 'store_true', dest = 'NBD_COW', help = 'Enable copy-on-write for the NBD device (Non-persistent changes)', default = NBD_COW) + nbd_group.add_argument('--nbd-cowinmem', action = 'store_true', dest = 'NBD_COWINMEM', help = 'Store copy-on-write pages in memory', default = NBD_COWINMEM) + nbd_group.add_argument('--nbd-copytoram', action = 'store_true', dest = 'NBD_COPYTORAM', help = 'Copy the NBD device to memory before serving clients', default = NBD_COPYTORAM) + nbd_group.add_argument('--nbd-server', action = 'store', dest = 'NBD_SERVER_IP', help = 'NBD Server IP', default = NBD_SERVER_IP) + nbd_group.add_argument('--nbd-port', action = 'store', dest = 'NBD_PORT', help = 'NBD Server Port', default = NBD_PORT) + # DHCP server arguments - exclusive = parser.add_mutually_exclusive_group(required = False) + dhcp_group = parser.add_argument_group(title = 'DHCP', description = 'Arguments relevant to the DHCP server') + exclusive = dhcp_group.add_mutually_exclusive_group(required = False) exclusive.add_argument('--dhcp', action = 'store_true', dest = 'USE_DHCP', help = 'Enable built-in DHCP server', default = SETTINGS['USE_DHCP']) exclusive.add_argument('--dhcp-proxy', action = 'store_true', dest = 'DHCP_MODE_PROXY', help = 'Enable built-in DHCP server in proxy mode (implies --dhcp)', default = SETTINGS['DHCP_MODE_PROXY']) - parser.add_argument('-s', '--dhcp-server-ip', action = 'store', dest = 'DHCP_SERVER_IP', help = 'DHCP Server IP', default = SETTINGS['DHCP_SERVER_IP']) - parser.add_argument('-p', '--dhcp-server-port', action = 'store', dest = 'DHCP_SERVER_PORT', help = 'DHCP Server Port', default = SETTINGS['DHCP_SERVER_PORT']) - parser.add_argument('-b', '--dhcp-begin', action = 'store', dest = 'DHCP_OFFER_BEGIN', help = 'DHCP lease range start', default = SETTINGS['DHCP_OFFER_BEGIN']) - parser.add_argument('-e', '--dhcp-end', action = 'store', dest = 'DHCP_OFFER_END', help = 'DHCP lease range end', default = SETTINGS['DHCP_OFFER_END']) - parser.add_argument('-n', '--dhcp-subnet', action = 'store', dest = 'DHCP_SUBNET', help = 'DHCP lease subnet', default = SETTINGS['DHCP_SUBNET']) - parser.add_argument('-r', '--dhcp-router', action = 'store', dest = 'DHCP_ROUTER', help = 'DHCP lease router', default = SETTINGS['DHCP_ROUTER']) - parser.add_argument('-d', '--dhcp-dns', action = 'store', dest = 'DHCP_DNS', help = 'DHCP lease DNS server', default = SETTINGS['DHCP_DNS']) - parser.add_argument('-c', '--dhcp-broadcast', action = 'store', dest = 'DHCP_BROADCAST', help = 'DHCP broadcast address', default = SETTINGS['DHCP_BROADCAST']) - parser.add_argument('-f', '--dhcp-fileserver', action = 'store', dest = 'DHCP_FILESERVER', help = 'DHCP fileserver IP', default = SETTINGS['DHCP_FILESERVER']) - parser.add_argument('--dhcp-whitelist', action = 'store_true', dest = 'DHCP_WHITELIST', help = 'Only respond to DHCP clients present in --static-config', default = False) + dhcp_group.add_argument('-s', '--dhcp-server-ip', action = 'store', dest = 'DHCP_SERVER_IP', help = 'DHCP Server IP', default = SETTINGS['DHCP_SERVER_IP']) + dhcp_group.add_argument('-p', '--dhcp-server-port', action = 'store', dest = 'DHCP_SERVER_PORT', help = 'DHCP Server Port', default = SETTINGS['DHCP_SERVER_PORT']) + dhcp_group.add_argument('-b', '--dhcp-begin', action = 'store', dest = 'DHCP_OFFER_BEGIN', help = 'DHCP lease range start', default = SETTINGS['DHCP_OFFER_BEGIN']) + dhcp_group.add_argument('-e', '--dhcp-end', action = 'store', dest = 'DHCP_OFFER_END', help = 'DHCP lease range end', default = SETTINGS['DHCP_OFFER_END']) + dhcp_group.add_argument('-n', '--dhcp-subnet', action = 'store', dest = 'DHCP_SUBNET', help = 'DHCP lease subnet', default = SETTINGS['DHCP_SUBNET']) + dhcp_group.add_argument('-r', '--dhcp-router', action = 'store', dest = 'DHCP_ROUTER', help = 'DHCP lease router', default = SETTINGS['DHCP_ROUTER']) + dhcp_group.add_argument('-d', '--dhcp-dns', action = 'store', dest = 'DHCP_DNS', help = 'DHCP lease DNS server', default = SETTINGS['DHCP_DNS']) + dhcp_group.add_argument('-c', '--dhcp-broadcast', action = 'store', dest = 'DHCP_BROADCAST', help = 'DHCP broadcast address', default = SETTINGS['DHCP_BROADCAST']) + dhcp_group.add_argument('-f', '--dhcp-fileserver', action = 'store', dest = 'DHCP_FILESERVER', help = 'DHCP fileserver IP', default = SETTINGS['DHCP_FILESERVER']) + dhcp_group.add_argument('--dhcp-whitelist', action = 'store_true', dest = 'DHCP_WHITELIST', help = 'Only respond to DHCP clients present in --static-config', default = False) # network boot directory and file name arguments parser.add_argument('-a', '--netboot-dir', action = 'store', dest = 'NETBOOT_DIR', help = 'Local file serve directory', default = SETTINGS['NETBOOT_DIR']) @@ -70,6 +88,8 @@ def parse_cli_arguments(): return parser.parse_args() +#NBD Default Server Settings + if __name__ == '__main__': try: # warn the user that they are starting PyPXE as non-root user @@ -138,7 +158,13 @@ def parse_cli_arguments(): else: args.NETBOOT_FILE = 'boot.http.ipxe' - # serve all files from one directory + if args.NBD_WRITE and not args.NBD_COW: + sys_logger.warning('NBD Write enabled but copy-on-write is not. Multiple clients may cause corruption') + + if args.NBD_COWINMEM or args.NBD_COPYTORAM: + sys_logger.warning('NBD cowinmem and copytoram can cause high RAM usage') + + #serve all files from one directory os.chdir (args.NETBOOT_DIR) # make a list of running threads for each service @@ -206,6 +232,29 @@ def parse_cli_arguments(): httpd.start() running_services.append(httpd) + #configure/start NBD server + if args.NBD_BLOCKDEVICE: + #setup nbd logger + nbd_logger = sys_logger.getChild('NBD') + sys_logger.info('Starting NBD server...') + nbdServer = nbd.NBD( + blockdevice = args.NBD_BLOCKDEVICE, + write = args.NBD_WRITE, + cow = args.NBD_COW, + inmem = args.NBD_COWINMEM, + copytoram = args.NBD_COPYTORAM, + ip = args.NBD_SERVER_IP, + port = args.NBD_PORT, + mode_debug = ("nbd" in args.MODE_DEBUG.lower() or "all" in args.MODE_DEBUG.lower()), + logger = nbd_logger + ) + nbd = threading.Thread(target = nbdServer.listen) + nbd.daemon = True + nbd.start() + runningServices.append(nbd) + + + sys_logger.info('PyPXE successfully initialized and running!') while map(lambda x: x.isAlive(), running_services): diff --git a/pypxe/nbd/__init__.py b/pypxe/nbd/__init__.py new file mode 100644 index 0000000..e43be03 --- /dev/null +++ b/pypxe/nbd/__init__.py @@ -0,0 +1 @@ +__version__ = '1.0' \ No newline at end of file diff --git a/pypxe/nbd/nbd.py b/pypxe/nbd/nbd.py index 7c5222b..1defe3a 100644 --- a/pypxe/nbd/nbd.py +++ b/pypxe/nbd/nbd.py @@ -27,6 +27,9 @@ def __init__(self, **serverSettings): handler.setFormatter(formatter) self.logger.addHandler(handler) + if self.mode_debug: + self.logger.setLevel(logging.DEBUG) + self.logger.debug('NOTICE: NBD server started in debug mode. NBD server is using the following:') self.logger.debug(' NBD Server IP: {}'.format(self.ip)) self.logger.debug(' NBD Server Port: {}'.format(self.port)) @@ -107,7 +110,11 @@ def handleClient(self, conn, addr, seeklock): while True: conn.recv(4) - [opcode, handle, offset, length] = struct.unpack("!IQQI", conn.recv(24, socket.MSG_WAITALL)) + try: + [opcode, handle, offset, length] = struct.unpack("!IQQI", conn.recv(24, socket.MSG_WAITALL)) + except struct.error: + #Client sent us something malformed, or gave up (module not loaded) + continue if opcode not in (0, 1, 2): # NBD_REP_ERR_UNSUP self.sendreply(conn, addr, 2**31+1, '') diff --git a/pypxe/nbd/writes.py b/pypxe/nbd/writes.py index 616099b..b1578d9 100644 --- a/pypxe/nbd/writes.py +++ b/pypxe/nbd/writes.py @@ -27,6 +27,7 @@ def basepages(self, offset, length): def read(self, offset, length): basepages = self.basepages(offset, length) + # this probably wants to be 2nd debug level self.logger.debug('%s reading %d bytes from %s. Pages: %s', self.addr, length, hex(offset), len(basepages)) data = "" @@ -51,6 +52,7 @@ def read(self, offset, length): def write(self, offset, data): basepages = self.basepages(offset, len(data)) + # this probably wants to be 2nd debug level self.logger.debug('%s writing %d bytes to %s. Pages: %s', self.addr, len(data), hex(offset), len(basepages)) for major, minor, length in basepages: @@ -115,6 +117,7 @@ def __init__(self, addr, imagefd, logger, seeklock): self.logger.debug('File for %s', addr) def read(self, offset, length): + # this probably wants to be 2nd debug level self.logger.debug('%s reading %d bytes from %s', self.addr, length, hex(offset)) # see COW.read() for lock reason self.seeklock.acquire() @@ -124,6 +127,7 @@ def read(self, offset, length): return data def write(self, offset, data): + # this probably wants to be 2nd debug level self.logger.debug('%s writing %d bytes to %s', self.addr, len(data), hex(offset)) self.seeklock.acquire() self.imagefd.seek(offset) From f3f46a130ddbda0408c8630200b76df9e1817fb8 Mon Sep 17 00:00:00 2001 From: PsychoMario Date: Sun, 19 Apr 2015 17:58:34 +0100 Subject: [PATCH 15/39] removed nbd test.py, added help to README.md --- README.md | 13 ++++++++++++- pypxe/nbd/test.py | 13 ------------- 2 files changed, 12 insertions(+), 14 deletions(-) delete mode 100644 pypxe/nbd/test.py diff --git a/README.md b/README.md index ab91e2b..adec4f6 100644 --- a/README.md +++ b/README.md @@ -77,7 +77,18 @@ The following are arguments that can be passed to `pypxe-server.py` when running |__`-a NETBOOT_DIR`__ or __`--netboot-dir NETBOOT_DIR`__|Specify the local directory where network boot files will be served|`'netboot'`| |__`-i NETBOOT_FILE`__ or __`--netboot-file NETBOOT_FILE`__|Specify the PXE boot file name|_automatically set based on what services are enabled or disabled, see [`DOCUMENTATION.md`](DOCUMENTATION.md) for further explanation_| + +##### Network Block Device Arguments +|Argument|Description|Default| +|__`--nbd NBD_BLOCKDEVICE`__|Specify the block device to be served by NBD and enable NBD. This can be a disk image.|`''`| +|__`--nbd-write`__|Open the block device for write access. UNSAFE: Multiple clients can cause corruption|`False`| +|__`--nbd-cow`__|When write is enabled, create a *volatile* file per client with their changes. Clients can write but changes are not shared or kept.|`True (Only applies if write is on)`| +|__`--nbd-cowinmem`__|Client volatile changes are stored in RAM rather than on disk. WARNING: High RAM usage (up to sizeof(block device)*clients)|`False`| +|__`--nbd-copytoram`__|Disk image is copied to RAM on start to speed up access. Changes are lost when write is used without cow.|`False`| +|__`--nbd-server`__|The NBD server IP address to bind to|`0.0.0.0`| +|__`--nbd-port`__|The NBD server port to bind to|`10809`| + ## Notes * `Core.iso` located in `netboot` is from the [TinyCore Project](http://distro.ibiblio.org/tinycorelinux/) and is provided as an example to network boot from using PyPXE -* `chainload.kpxe` located in `netboot` is the `undionly.kpxe` from the [iPXE Project](http://ipxe.org/) +* `chainload.kpxe` located in `netboot` is the `undionly.kpxe` from the [iPXE Project](http://ipxe.org/) * `ldlinux.c32`, `libutil.c32`, `pxelinux.0`, `menu.c32`, and `memdisk` located in `netboot` are from the [SYSLINUX Project](http://www.syslinux.org/) version [6.02](http://www.syslinux.org/wiki/index.php/Syslinux_6_Changelog#Changes_in_6.02) diff --git a/pypxe/nbd/test.py b/pypxe/nbd/test.py deleted file mode 100644 index 0478f2a..0000000 --- a/pypxe/nbd/test.py +++ /dev/null @@ -1,13 +0,0 @@ -import nbd -args = { - 'blockdevice':'arch.img', - 'mode_debug':True, - 'write':True, - 'cow':True, - 'inmem':True, - 'copytoram':True - } -# rw and multiple clients not safe -# rcow/ro and multiple clients OK - -nbd.NBD(**args).listen() From 901b3f68ae9751726ac0c5280c0c9ad880daca54 Mon Sep 17 00:00:00 2001 From: PsychoMario Date: Sun, 10 May 2015 23:39:04 +0100 Subject: [PATCH 16/39] reorder nbd below dhcp options --- pypxe-server.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/pypxe-server.py b/pypxe-server.py index db49061..34b2321 100755 --- a/pypxe-server.py +++ b/pypxe-server.py @@ -56,15 +56,6 @@ def parse_cli_arguments(): parser.add_argument('--syslog', action = 'store', dest = 'SYSLOG_SERVER', help = 'Syslog server', default = SETTINGS['SYSLOG_SERVER']) parser.add_argument('--syslog-port', action = 'store', dest = 'SYSLOG_PORT', help = 'Syslog server port', default = SETTINGS['SYSLOG_PORT']) - # NBD server arguments - nbd_group = parser.add_argument_group(title = 'Network Block Device', description = 'Arguments relevant to the NBD server') - nbd_group.add_argument('--nbd', action = 'store', dest = 'NBD_BLOCKDEVICE', help = 'Enable the NDB server with a specific block device (Can be a disk image)', default = NBD_BLOCKDEVICE) - nbd_group.add_argument('--nbd-write', action = 'store_true', dest = 'NBD_WRITE', help = 'Enable writes on the NBD device', default = NBD_WRITE) - nbd_group.add_argument('--nbd-cow', action = 'store_true', dest = 'NBD_COW', help = 'Enable copy-on-write for the NBD device (Non-persistent changes)', default = NBD_COW) - nbd_group.add_argument('--nbd-cowinmem', action = 'store_true', dest = 'NBD_COWINMEM', help = 'Store copy-on-write pages in memory', default = NBD_COWINMEM) - nbd_group.add_argument('--nbd-copytoram', action = 'store_true', dest = 'NBD_COPYTORAM', help = 'Copy the NBD device to memory before serving clients', default = NBD_COPYTORAM) - nbd_group.add_argument('--nbd-server', action = 'store', dest = 'NBD_SERVER_IP', help = 'NBD Server IP', default = NBD_SERVER_IP) - nbd_group.add_argument('--nbd-port', action = 'store', dest = 'NBD_PORT', help = 'NBD Server Port', default = NBD_PORT) # DHCP server arguments dhcp_group = parser.add_argument_group(title = 'DHCP', description = 'Arguments relevant to the DHCP server') @@ -86,6 +77,16 @@ def parse_cli_arguments(): parser.add_argument('-a', '--netboot-dir', action = 'store', dest = 'NETBOOT_DIR', help = 'Local file serve directory', default = SETTINGS['NETBOOT_DIR']) parser.add_argument('-i', '--netboot-file', action = 'store', dest = 'NETBOOT_FILE', help = 'PXE boot file name (after iPXE if --ipxe)', default = SETTINGS['NETBOOT_FILE']) + # NBD server arguments + nbd_group = parser.add_argument_group(title = 'Network Block Device', description = 'Arguments relevant to the NBD server') + nbd_group.add_argument('--nbd', action = 'store', dest = 'NBD_BLOCKDEVICE', help = 'Enable the NDB server with a specific block device (Can be a disk image)', default = SETTINGS['NBD_BLOCKDEVICE']) + nbd_group.add_argument('--nbd-write', action = 'store_true', dest = 'NBD_WRITE', help = 'Enable writes on the NBD device', default = SETTINGS['NBD_WRITE']) + nbd_group.add_argument('--nbd-cow', action = 'store_true', dest = 'NBD_COW', help = 'Enable copy-on-write for the NBD device (Non-persistent changes)', default = SETTINGS['NBD_COW']) + nbd_group.add_argument('--nbd-cowinmem', action = 'store_true', dest = 'NBD_COWINMEM', help = 'Store copy-on-write pages in memory', default = SETTINGS['NBD_COWINMEM']) + nbd_group.add_argument('--nbd-copytoram', action = 'store_true', dest = 'NBD_COPYTORAM', help = 'Copy the NBD device to memory before serving clients', default = SETTINGS['NBD_COPYTORAM']) + nbd_group.add_argument('--nbd-server', action = 'store', dest = 'NBD_SERVER_IP', help = 'NBD Server IP', default = SETTINGS['NBD_SERVER_IP']) + nbd_group.add_argument('--nbd-port', action = 'store', dest = 'NBD_PORT', help = 'NBD Server Port', default = SETTINGS['NBD_PORT']) + return parser.parse_args() #NBD Default Server Settings From b18c795daba9596404562821a76589085687dd23 Mon Sep 17 00:00:00 2001 From: PsychoMario Date: Sun, 10 May 2015 23:41:02 +0100 Subject: [PATCH 17/39] removed out of place comment --- pypxe-server.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/pypxe-server.py b/pypxe-server.py index 34b2321..b7f9843 100755 --- a/pypxe-server.py +++ b/pypxe-server.py @@ -89,8 +89,6 @@ def parse_cli_arguments(): return parser.parse_args() -#NBD Default Server Settings - if __name__ == '__main__': try: # warn the user that they are starting PyPXE as non-root user From 6b3dff0148a760be102179964aa0554cb66c0b76 Mon Sep 17 00:00:00 2001 From: PsychoMario Date: Sun, 10 May 2015 23:52:00 +0100 Subject: [PATCH 18/39] fix import, logging --- pypxe-server.py | 6 +++--- pypxe/nbd/__init__.py | 3 ++- pypxe/nbd/nbd.py | 10 +++++----- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/pypxe-server.py b/pypxe-server.py index b7f9843..bd5e016 100755 --- a/pypxe-server.py +++ b/pypxe-server.py @@ -15,6 +15,7 @@ from pypxe import tftp # PyPXE TFTP service from pypxe import dhcp # PyPXE DHCP service from pypxe import http # PyPXE HTTP service +from pypxe import nbd # PyPXE NBD service # default settings SETTINGS = {'NETBOOT_DIR':'netboot', @@ -245,12 +246,11 @@ def parse_cli_arguments(): ip = args.NBD_SERVER_IP, port = args.NBD_PORT, mode_debug = ("nbd" in args.MODE_DEBUG.lower() or "all" in args.MODE_DEBUG.lower()), - logger = nbd_logger - ) + logger = nbd_logger) nbd = threading.Thread(target = nbdServer.listen) nbd.daemon = True nbd.start() - runningServices.append(nbd) + running_services.append(nbd) diff --git a/pypxe/nbd/__init__.py b/pypxe/nbd/__init__.py index e43be03..513fc2b 100644 --- a/pypxe/nbd/__init__.py +++ b/pypxe/nbd/__init__.py @@ -1 +1,2 @@ -__version__ = '1.0' \ No newline at end of file +__version__ = '1.0' +from nbd import * diff --git a/pypxe/nbd/nbd.py b/pypxe/nbd/nbd.py index 1defe3a..352df96 100644 --- a/pypxe/nbd/nbd.py +++ b/pypxe/nbd/nbd.py @@ -31,11 +31,11 @@ def __init__(self, **serverSettings): self.logger.setLevel(logging.DEBUG) self.logger.debug('NOTICE: NBD server started in debug mode. NBD server is using the following:') - self.logger.debug(' NBD Server IP: {}'.format(self.ip)) - self.logger.debug(' NBD Server Port: {}'.format(self.port)) - self.logger.debug(' NBD Block Device: {}'.format(self.bd)) - self.logger.debug(' NBD Block Device Writes: {}'.format(self.write)) - self.logger.debug(' NBD Block Write Method: {} ({})'.format("Copy-On-Write" if self.cow else "File", "Memory" if self.inmem else "Disk")) + self.logger.debug('NBD Server IP: {}'.format(self.ip)) + self.logger.debug('NBD Server Port: {}'.format(self.port)) + self.logger.debug('NBD Block Device: {}'.format(self.bd)) + self.logger.debug('NBD Block Device Writes: {}'.format(self.write)) + self.logger.debug('NBD Block Write Method: {} ({})'.format("Copy-On-Write" if self.cow else "File", "Memory" if self.inmem else "Disk")) if self.mode_debug: self.logger.setLevel(logging.DEBUG) From d40805b399c34c5294a7448a0800b08930c4b083 Mon Sep 17 00:00:00 2001 From: PsychoMario Date: Mon, 11 May 2015 18:29:21 +0100 Subject: [PATCH 19/39] Disabling services on the CLI. fixes #83 --- pypxe-server.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/pypxe-server.py b/pypxe-server.py index bd5e016..a9894e6 100755 --- a/pypxe-server.py +++ b/pypxe-server.py @@ -48,9 +48,19 @@ def parse_cli_arguments(): # main service arguments parser = argparse.ArgumentParser(description = 'Set options at runtime. Defaults are in %(prog)s', formatter_class = argparse.ArgumentDefaultsHelpFormatter) - parser.add_argument('--ipxe', action = 'store_true', dest = 'USE_IPXE', help = 'Enable iPXE ROM', default = SETTINGS['USE_IPXE']) - parser.add_argument('--http', action = 'store_true', dest = 'USE_HTTP', help = 'Enable built-in HTTP server', default = SETTINGS['USE_HTTP']) - parser.add_argument('--no-tftp', action = 'store_false', dest = 'USE_TFTP', help = 'Disable built-in TFTP server, by default it is enabled', default = SETTINGS['USE_TFTP']) + + ipxeexclusive = parser.add_mutually_exclusive_group(required = False) + ipxeexclusive.add_argument('--ipxe', action = 'store_true', dest = 'USE_IPXE', help = 'Enable iPXE ROM', default = SETTINGS['USE_IPXE']) + ipxeexclusive.add_argument('--no-ipxe', action = 'store_false', dest = 'USE_IPXE', help = 'Disable iPXE ROM', default = not SETTINGS['USE_IPXE']) + + httpexclusive = parser.add_mutually_exclusive_group(required = False) + httpexclusive.add_argument('--http', action = 'store_true', dest = 'USE_HTTP', help = 'Disable built-in HTTP server', default = SETTINGS['USE_HTTP']) + httpexclusive.add_argument('--no-http', action = 'store_false', dest = 'USE_HTTP', help = 'Enable built-in HTTP server', default = not SETTINGS['USE_HTTP']) + + tftpexclusive = parser.add_mutually_exclusive_group(required = False) + tftpexclusive.add_argument('--tftp', action = 'store_true', dest = 'USE_TFTP', help = 'Enable built-in TFTP server, by default it is enabled', default = SETTINGS['USE_TFTP']) + tftpexclusive.add_argument('--no-tftp', action = 'store_false', dest = 'USE_TFTP', help = 'Disable built-in TFTP server, by default it is enabled', default = not SETTINGS['USE_TFTP']) + parser.add_argument('--debug', action = 'store', dest = 'MODE_DEBUG', help = 'Comma Seperated (http,tftp,dhcp). Adds verbosity to the selected services while they run. Use \'all\' for enabling debug on all services', default = SETTINGS['MODE_DEBUG']) parser.add_argument('--config', action = 'store', dest = 'JSON_CONFIG', help = 'Configure from a JSON file rather than the command line', default = '') parser.add_argument('--static-config', action = 'store', dest = 'STATIC_CONFIG', help = 'Configure leases from a json file rather than the command line', default = '') @@ -61,8 +71,10 @@ def parse_cli_arguments(): # DHCP server arguments dhcp_group = parser.add_argument_group(title = 'DHCP', description = 'Arguments relevant to the DHCP server') exclusive = dhcp_group.add_mutually_exclusive_group(required = False) + exclusive.add_argument('--no-dhcp', action = 'store_false', dest = 'USE_DHCP', help = 'Disable built-in DHCP server', default = not SETTINGS['USE_DHCP']) exclusive.add_argument('--dhcp', action = 'store_true', dest = 'USE_DHCP', help = 'Enable built-in DHCP server', default = SETTINGS['USE_DHCP']) exclusive.add_argument('--dhcp-proxy', action = 'store_true', dest = 'DHCP_MODE_PROXY', help = 'Enable built-in DHCP server in proxy mode (implies --dhcp)', default = SETTINGS['DHCP_MODE_PROXY']) + dhcp_group.add_argument('-s', '--dhcp-server-ip', action = 'store', dest = 'DHCP_SERVER_IP', help = 'DHCP Server IP', default = SETTINGS['DHCP_SERVER_IP']) dhcp_group.add_argument('-p', '--dhcp-server-port', action = 'store', dest = 'DHCP_SERVER_PORT', help = 'DHCP Server Port', default = SETTINGS['DHCP_SERVER_PORT']) dhcp_group.add_argument('-b', '--dhcp-begin', action = 'store', dest = 'DHCP_OFFER_BEGIN', help = 'DHCP lease range start', default = SETTINGS['DHCP_OFFER_BEGIN']) From f9533efdf77fc8118a13defe58f5aadbd8167f4b Mon Sep 17 00:00:00 2001 From: PsychoMario Date: Mon, 11 May 2015 18:35:33 +0100 Subject: [PATCH 20/39] Allow negate debug args. Fixes #82 --- pypxe-server.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/pypxe-server.py b/pypxe-server.py index a9894e6..f78be79 100755 --- a/pypxe-server.py +++ b/pypxe-server.py @@ -102,6 +102,12 @@ def parse_cli_arguments(): return parser.parse_args() +def do_debug(service): + return ((service in args.MODE_DEBUG.lower() + or 'all' in args.MODE_DEBUG.lower()) + and '-{0}'.format(service) not in args.MODE_DEBUG.lower()) + + if __name__ == '__main__': try: # warn the user that they are starting PyPXE as non-root user @@ -190,7 +196,7 @@ def parse_cli_arguments(): sys_logger.info('Starting TFTP server...') # setup the thread - tftp_server = tftp.TFTPD(mode_debug = ('tftp' in args.MODE_DEBUG.lower() or 'all' in args.MODE_DEBUG.lower()), logger = tftp_logger) + tftp_server = tftp.TFTPD(mode_debug = do_debug('tftp'), logger = tftp_logger) tftpd = threading.Thread(target = tftp_server.listen) tftpd.daemon = True tftpd.start() @@ -221,10 +227,10 @@ def parse_cli_arguments(): use_ipxe = args.USE_IPXE, use_http = args.USE_HTTP, mode_proxy = args.DHCP_MODE_PROXY, - mode_debug = ('dhcp' in args.MODE_DEBUG.lower() or 'all' in args.MODE_DEBUG.lower()), + mode_debug = do_debug('dhcp'), whitelist = args.DHCP_WHITELIST, - logger = dhcp_logger, - static_config = loaded_statics) + static_config = loaded_statics, + logger = dhcp_logger) dhcpd = threading.Thread(target = dhcp_server.listen) dhcpd.daemon = True dhcpd.start() @@ -238,7 +244,7 @@ def parse_cli_arguments(): sys_logger.info('Starting HTTP server...') # setup the thread - http_server = http.HTTPD(mode_debug = ('http' in args.MODE_DEBUG.lower() or 'all' in args.MODE_DEBUG.lower()), logger = http_logger) + http_server = http.HTTPD(mode_debug = do_debug('http'), logger = http_logger) httpd = threading.Thread(target = http_server.listen) httpd.daemon = True httpd.start() From 7c29ff9cfe0f5d54a2c9353f689aca9c20803e90 Mon Sep 17 00:00:00 2001 From: PsychoMario Date: Mon, 11 May 2015 19:26:23 +0100 Subject: [PATCH 21/39] Remove Short Options. Fixes #85 --- pypxe-server.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/pypxe-server.py b/pypxe-server.py index f78be79..28cd5af 100755 --- a/pypxe-server.py +++ b/pypxe-server.py @@ -75,20 +75,20 @@ def parse_cli_arguments(): exclusive.add_argument('--dhcp', action = 'store_true', dest = 'USE_DHCP', help = 'Enable built-in DHCP server', default = SETTINGS['USE_DHCP']) exclusive.add_argument('--dhcp-proxy', action = 'store_true', dest = 'DHCP_MODE_PROXY', help = 'Enable built-in DHCP server in proxy mode (implies --dhcp)', default = SETTINGS['DHCP_MODE_PROXY']) - dhcp_group.add_argument('-s', '--dhcp-server-ip', action = 'store', dest = 'DHCP_SERVER_IP', help = 'DHCP Server IP', default = SETTINGS['DHCP_SERVER_IP']) - dhcp_group.add_argument('-p', '--dhcp-server-port', action = 'store', dest = 'DHCP_SERVER_PORT', help = 'DHCP Server Port', default = SETTINGS['DHCP_SERVER_PORT']) - dhcp_group.add_argument('-b', '--dhcp-begin', action = 'store', dest = 'DHCP_OFFER_BEGIN', help = 'DHCP lease range start', default = SETTINGS['DHCP_OFFER_BEGIN']) - dhcp_group.add_argument('-e', '--dhcp-end', action = 'store', dest = 'DHCP_OFFER_END', help = 'DHCP lease range end', default = SETTINGS['DHCP_OFFER_END']) - dhcp_group.add_argument('-n', '--dhcp-subnet', action = 'store', dest = 'DHCP_SUBNET', help = 'DHCP lease subnet', default = SETTINGS['DHCP_SUBNET']) - dhcp_group.add_argument('-r', '--dhcp-router', action = 'store', dest = 'DHCP_ROUTER', help = 'DHCP lease router', default = SETTINGS['DHCP_ROUTER']) - dhcp_group.add_argument('-d', '--dhcp-dns', action = 'store', dest = 'DHCP_DNS', help = 'DHCP lease DNS server', default = SETTINGS['DHCP_DNS']) - dhcp_group.add_argument('-c', '--dhcp-broadcast', action = 'store', dest = 'DHCP_BROADCAST', help = 'DHCP broadcast address', default = SETTINGS['DHCP_BROADCAST']) - dhcp_group.add_argument('-f', '--dhcp-fileserver', action = 'store', dest = 'DHCP_FILESERVER', help = 'DHCP fileserver IP', default = SETTINGS['DHCP_FILESERVER']) + dhcp_group.add_argument('--dhcp-server-ip', action = 'store', dest = 'DHCP_SERVER_IP', help = 'DHCP Server IP', default = SETTINGS['DHCP_SERVER_IP']) + dhcp_group.add_argument('--dhcp-server-port', action = 'store', dest = 'DHCP_SERVER_PORT', help = 'DHCP Server Port', default = SETTINGS['DHCP_SERVER_PORT']) + dhcp_group.add_argument('--dhcp-begin', action = 'store', dest = 'DHCP_OFFER_BEGIN', help = 'DHCP lease range start', default = SETTINGS['DHCP_OFFER_BEGIN']) + dhcp_group.add_argument('--dhcp-end', action = 'store', dest = 'DHCP_OFFER_END', help = 'DHCP lease range end', default = SETTINGS['DHCP_OFFER_END']) + dhcp_group.add_argument('--dhcp-subnet', action = 'store', dest = 'DHCP_SUBNET', help = 'DHCP lease subnet', default = SETTINGS['DHCP_SUBNET']) + dhcp_group.add_argument('--dhcp-router', action = 'store', dest = 'DHCP_ROUTER', help = 'DHCP lease router', default = SETTINGS['DHCP_ROUTER']) + dhcp_group.add_argument('--dhcp-dns', action = 'store', dest = 'DHCP_DNS', help = 'DHCP lease DNS server', default = SETTINGS['DHCP_DNS']) + dhcp_group.add_argument('--dhcp-broadcast', action = 'store', dest = 'DHCP_BROADCAST', help = 'DHCP broadcast address', default = SETTINGS['DHCP_BROADCAST']) + dhcp_group.add_argument('--dhcp-fileserver', action = 'store', dest = 'DHCP_FILESERVER', help = 'DHCP fileserver IP', default = SETTINGS['DHCP_FILESERVER']) dhcp_group.add_argument('--dhcp-whitelist', action = 'store_true', dest = 'DHCP_WHITELIST', help = 'Only respond to DHCP clients present in --static-config', default = False) # network boot directory and file name arguments - parser.add_argument('-a', '--netboot-dir', action = 'store', dest = 'NETBOOT_DIR', help = 'Local file serve directory', default = SETTINGS['NETBOOT_DIR']) - parser.add_argument('-i', '--netboot-file', action = 'store', dest = 'NETBOOT_FILE', help = 'PXE boot file name (after iPXE if --ipxe)', default = SETTINGS['NETBOOT_FILE']) + parser.add_argument('--netboot-dir', action = 'store', dest = 'NETBOOT_DIR', help = 'Local file serve directory', default = SETTINGS['NETBOOT_DIR']) + parser.add_argument('--netboot-file', action = 'store', dest = 'NETBOOT_FILE', help = 'PXE boot file name (after iPXE if --ipxe)', default = SETTINGS['NETBOOT_FILE']) # NBD server arguments nbd_group = parser.add_argument_group(title = 'Network Block Device', description = 'Arguments relevant to the NBD server') From e384c217a51bdb6106af70876a43eb1f69769469 Mon Sep 17 00:00:00 2001 From: Michael Mattioli Date: Mon, 11 May 2015 16:46:59 -0400 Subject: [PATCH 22/39] version bump and rename of example DHCP leases JSON renamed example DHCP leases JSON file modified package version in __init__.py --- example-leases.json => example_dhcp_leases.json | 0 pypxe/__init__.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename example-leases.json => example_dhcp_leases.json (100%) diff --git a/example-leases.json b/example_dhcp_leases.json similarity index 100% rename from example-leases.json rename to example_dhcp_leases.json diff --git a/pypxe/__init__.py b/pypxe/__init__.py index fcb6b5d..6d5e09d 100644 --- a/pypxe/__init__.py +++ b/pypxe/__init__.py @@ -1 +1 @@ -__version__ = '1.5' +__version__ = '1.6' From e914becfb6b75c051c2e0ce2df9aa9203dcc5143 Mon Sep 17 00:00:00 2001 From: Michael Mattioli Date: Mon, 11 May 2015 17:47:22 -0400 Subject: [PATCH 23/39] renamed variables, adjusted spacing and string quotations, adjusted comments removed version number from NBD package because it was unnecessary to have changed all strings to single-quoted strings added/removed whitespace as necessary to conform to PEP8 renamed variables to conform with PEP8 adjusted comments for easier understanding removed unnecessary comments moved to str.format() throughout --- pypxe-server.py | 77 ++++++++++++++++---------------- pypxe/dhcp.py | 51 +++++++++++---------- pypxe/nbd/__init__.py | 1 - pypxe/nbd/nbd.py | 101 +++++++++++++++++++++--------------------- pypxe/nbd/writes.py | 83 ++++++++++++++++------------------ pypxe/tftp.py | 6 +-- 6 files changed, 154 insertions(+), 165 deletions(-) diff --git a/pypxe-server.py b/pypxe-server.py index 28cd5af..10620a7 100755 --- a/pypxe-server.py +++ b/pypxe-server.py @@ -36,11 +36,11 @@ 'USE_TFTP':True, 'USE_DHCP':True, 'DHCP_MODE_PROXY':False, - 'NBD_BLOCKDEVICE':'', + 'NBD_BLOCK_DEVICE':'', 'NBD_WRITE':False, 'NBD_COW':True, - 'NBD_COWINMEM':False, - 'NBD_COPYTORAM':False, + 'NBD_COW_IN_MEM':False, + 'NBD_COPY_TO_RAM':False, 'NBD_SERVER_IP':'0.0.0.0', 'NBD_PORT':10809, 'MODE_DEBUG':''} @@ -92,11 +92,11 @@ def parse_cli_arguments(): # NBD server arguments nbd_group = parser.add_argument_group(title = 'Network Block Device', description = 'Arguments relevant to the NBD server') - nbd_group.add_argument('--nbd', action = 'store', dest = 'NBD_BLOCKDEVICE', help = 'Enable the NDB server with a specific block device (Can be a disk image)', default = SETTINGS['NBD_BLOCKDEVICE']) + nbd_group.add_argument('--nbd', action = 'store', dest = 'NBD_BLOCK_DEVICE', help = 'Enable the NDB server with a specific block device (Can be a disk image)', default = SETTINGS['NBD_BLOCK_DEVICE']) nbd_group.add_argument('--nbd-write', action = 'store_true', dest = 'NBD_WRITE', help = 'Enable writes on the NBD device', default = SETTINGS['NBD_WRITE']) nbd_group.add_argument('--nbd-cow', action = 'store_true', dest = 'NBD_COW', help = 'Enable copy-on-write for the NBD device (Non-persistent changes)', default = SETTINGS['NBD_COW']) - nbd_group.add_argument('--nbd-cowinmem', action = 'store_true', dest = 'NBD_COWINMEM', help = 'Store copy-on-write pages in memory', default = SETTINGS['NBD_COWINMEM']) - nbd_group.add_argument('--nbd-copytoram', action = 'store_true', dest = 'NBD_COPYTORAM', help = 'Copy the NBD device to memory before serving clients', default = SETTINGS['NBD_COPYTORAM']) + nbd_group.add_argument('--nbd-cow-in-mem', action = 'store_true', dest = 'NBD_COW_IN_MEM', help = 'Store copy-on-write pages in memory', default = SETTINGS['NBD_COW_IN_MEM']) + nbd_group.add_argument('--nbd-copy-to-ram', action = 'store_true', dest = 'NBD_COPY_TO_RAM', help = 'Copy the NBD device to memory before serving clients', default = SETTINGS['NBD_COPY_TO_RAM']) nbd_group.add_argument('--nbd-server', action = 'store', dest = 'NBD_SERVER_IP', help = 'NBD Server IP', default = SETTINGS['NBD_SERVER_IP']) nbd_group.add_argument('--nbd-port', action = 'store', dest = 'NBD_PORT', help = 'NBD Server Port', default = SETTINGS['NBD_PORT']) @@ -107,7 +107,6 @@ def do_debug(service): or 'all' in args.MODE_DEBUG.lower()) and '-{0}'.format(service) not in args.MODE_DEBUG.lower()) - if __name__ == '__main__': try: # warn the user that they are starting PyPXE as non-root user @@ -214,23 +213,23 @@ def do_debug(service): # setup the thread dhcp_server = dhcp.DHCPD( - ip = args.DHCP_SERVER_IP, - port = args.DHCP_SERVER_PORT, - offer_from = args.DHCP_OFFER_BEGIN, - offer_to = args.DHCP_OFFER_END, - subnet_mask = args.DHCP_SUBNET, - router = args.DHCP_ROUTER, - dns_server = args.DHCP_DNS, - broadcast = args.DHCP_BROADCAST, - file_server = args.DHCP_FILESERVER, - file_name = args.NETBOOT_FILE, - use_ipxe = args.USE_IPXE, - use_http = args.USE_HTTP, - mode_proxy = args.DHCP_MODE_PROXY, - mode_debug = do_debug('dhcp'), - whitelist = args.DHCP_WHITELIST, - static_config = loaded_statics, - logger = dhcp_logger) + ip = args.DHCP_SERVER_IP, + port = args.DHCP_SERVER_PORT, + offer_from = args.DHCP_OFFER_BEGIN, + offer_to = args.DHCP_OFFER_END, + subnet_mask = args.DHCP_SUBNET, + router = args.DHCP_ROUTER, + dns_server = args.DHCP_DNS, + broadcast = args.DHCP_BROADCAST, + file_server = args.DHCP_FILESERVER, + file_name = args.NETBOOT_FILE, + use_ipxe = args.USE_IPXE, + use_http = args.USE_HTTP, + mode_proxy = args.DHCP_MODE_PROXY, + mode_debug = do_debug('dhcp'), + whitelist = args.DHCP_WHITELIST, + static_config = loaded_statics, + logger = dhcp_logger) dhcpd = threading.Thread(target = dhcp_server.listen) dhcpd.daemon = True dhcpd.start() @@ -250,28 +249,26 @@ def do_debug(service): httpd.start() running_services.append(httpd) - #configure/start NBD server - if args.NBD_BLOCKDEVICE: - #setup nbd logger + # configure/start NBD server + if args.NBD_BLOCK_DEVICE: + # setup NBD logger nbd_logger = sys_logger.getChild('NBD') sys_logger.info('Starting NBD server...') - nbdServer = nbd.NBD( - blockdevice = args.NBD_BLOCKDEVICE, - write = args.NBD_WRITE, - cow = args.NBD_COW, - inmem = args.NBD_COWINMEM, - copytoram = args.NBD_COPYTORAM, - ip = args.NBD_SERVER_IP, - port = args.NBD_PORT, - mode_debug = ("nbd" in args.MODE_DEBUG.lower() or "all" in args.MODE_DEBUG.lower()), - logger = nbd_logger) - nbd = threading.Thread(target = nbdServer.listen) + nbd_server = nbd.NBD( + block_device = args.NBD_BLOCK_DEVICE, + write = args.NBD_WRITE, + cow = args.NBD_COW, + in_mem = args.NBD_COW_IN_MEM, + copy_to_ram = args.NBD_COPY_TO_RAM, + ip = args.NBD_SERVER_IP, + port = args.NBD_PORT, + mode_debug = ('nbd' in args.MODE_DEBUG.lower() or 'all' in args.MODE_DEBUG.lower()), + logger = nbd_logger) + nbd = threading.Thread(target = nbd_server.listen) nbd.daemon = True nbd.start() running_services.append(nbd) - - sys_logger.info('PyPXE successfully initialized and running!') while map(lambda x: x.isAlive(), running_services): diff --git a/pypxe/dhcp.py b/pypxe/dhcp.py index 691833e..6c5a593 100644 --- a/pypxe/dhcp.py +++ b/pypxe/dhcp.py @@ -14,6 +14,7 @@ class OutOfLeasesError(Exception): pass + class DHCPD: ''' This class implements a DHCP Server, limited to PXE options. @@ -86,8 +87,6 @@ def __init__(self, **server_settings): self.logger.debug('Using iPXE: {0}'.format(self.ipxe)) self.logger.debug('Using HTTP Server: {0}'.format(self.http)) - - self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) @@ -96,7 +95,7 @@ def __init__(self, **server_settings): # key is MAC self.leases = defaultdict(lambda: {'ip': '', 'expire': 0, 'ipxe': self.ipxe}) - def getNamespacedStatic(self, path, fallback = {}): + def get_namespaced_static(self, path, fallback = {}): statics = self.static_config for child in path.split('.'): statics = statics.get(child, {}) @@ -156,7 +155,7 @@ def tlv_parse(self, raw): ret[tag] = [value] return ret - def print_mac(self, mac): + def get_mac(self, mac): ''' This method converts the MAC Address from binary to human-readable format for logging. @@ -178,11 +177,11 @@ def craft_header(self, message): if self.leases[client_mac]['ip']: # OFFER offer = self.leases[client_mac]['ip'] else: # ACK - offer = self.getNamespacedStatic('dhcp.binding.{}.ipaddr'.format(self.print_mac(client_mac))) + offer = self.get_namespaced_static('dhcp.binding.{0}.ipaddr'.format(self.get_mac(client_mac))) offer = offer if offer else self.next_ip() self.leases[client_mac]['ip'] = offer self.leases[client_mac]['expire'] = time() + 86400 - self.logger.debug('New Assignment - MAC: {0} -> IP: {1}'.format(self.print_mac(client_mac), self.leases[client_mac]['ip'])) + self.logger.debug('New Assignment - MAC: {0} -> IP: {1}'.format(self.get_mac(client_mac), self.leases[client_mac]['ip'])) response += socket.inet_aton(offer) # yiaddr else: response += socket.inet_aton('0.0.0.0') @@ -206,19 +205,19 @@ def craft_options(self, opt53, client_mac): opt53: 2 - DHCPOFFER 5 - DHCPACK - (See RFC2132 9.6) + See RFC2132 9.6 for details. ''' response = self.tlv_encode(53, chr(opt53)) # message type, OFFER response += self.tlv_encode(54, socket.inet_aton(self.ip)) # DHCP Server if not self.mode_proxy: - subnetmask = self.getNamespacedStatic('dhcp.binding.{}.subnet'.format(self.print_mac(client_mac)), self.subnet_mask) - response += self.tlv_encode(1, socket.inet_aton(subnetmask)) #SubnetMask - router = self.getNamespacedStatic('dhcp.binding.{}.router'.format(self.print_mac(client_mac)), self.router) - response += self.tlv_encode(3, socket.inet_aton(router)) #Router - dnsserver = self.getNamespacedStatic('dhcp.binding.{}.dns'.format(self.print_mac(client_mac)), [self.dns_server]) + subnetmask = self.get_namespaced_static('dhcp.binding.{0}.subnet'.format(self.get_mac(client_mac)), self.subnet_mask) + response += self.tlv_encode(1, socket.inet_aton(subnetmask)) # subnet mask + router = self.get_namespaced_static('dhcp.binding.{0}.router'.format(self.get_mac(client_mac)), self.router) + response += self.tlv_encode(3, socket.inet_aton(router)) # router + dnsserver = self.get_namespaced_static('dhcp.binding.{0}.dns'.format(self.get_mac(client_mac)), [self.dns_server]) dnsserver = ''.join([socket.inet_aton(i) for i in dnsserver]) response += self.tlv_encode(6, dnsserver) - response += self.tlv_encode(51, struct.pack('!I', 86400)) #lease time + response += self.tlv_encode(51, struct.pack('!I', 86400)) # lease time # TFTP Server OR HTTP Server; if iPXE, need both response += self.tlv_encode(66, self.file_server) @@ -226,16 +225,16 @@ def craft_options(self, opt53, client_mac): # file_name null terminated if not self.ipxe or not self.leases[client_mac]['ipxe']: # http://www.syslinux.org/wiki/index.php/PXELINUX#UEFI - if 93 in self.leases[client_mac]["options"] and not self.force_file_name: - [arch] = struct.unpack("!H", self.leases[client_mac]["options"][93][0]) + if 93 in self.leases[client_mac]['options'] and not self.force_file_name: + [arch] = struct.unpack("!H", self.leases[client_mac]['options'][93][0]) if arch == 0: # BIOS/default - response += self.tlv_encode(67, "pxelinux.0" + chr(0)) + response += self.tlv_encode(67, 'pxelinux.0' + chr(0)) elif arch == 6: # EFI IA32 - response += self.tlv_encode(67, "syslinux.efi32" + chr(0)) - elif arch == 7: # EFI BC, x86-64 according to link above - response += self.tlv_encode(67, "syslinux.efi64" + chr(0)) + response += self.tlv_encode(67, 'syslinux.efi32' + chr(0)) + elif arch == 7: # EFI BC, x86-64 (according to the above link) + response += self.tlv_encode(67, 'syslinux.efi64' + chr(0)) elif arch == 9: # EFI x86-64 - response += self.tlv_encode(67, "syslinux.efi64" + chr(0)) + response += self.tlv_encode(67, 'syslinux.efi64' + chr(0)) else: response += self.tlv_encode(67, self.file_name + chr(0)) else: @@ -284,10 +283,10 @@ def dhcp_ack(self, message): def validate_req(self, client_mac): # client request is valid only if contains Vendor-Class = PXEClient - if self.whitelist and self.print_mac(client_mac) not in self.getNamespacedStatic('dhcp.binding'): + if self.whitelist and self.get_mac(client_mac) not in self.get_namespaced_static('dhcp.binding'): self.logger.debug('Non-whitelisted client request received') return False - if 60 in self.leases[client_mac]["options"] and 'PXEClient' in self.leases[client_mac]["options"][60][0]: + if 60 in self.leases[client_mac]['options'] and 'PXEClient' in self.leases[client_mac]['options'][60][0]: self.logger.debug('PXE client request received') return True if self.mode_debug: @@ -303,20 +302,20 @@ def listen(self): self.logger.debug('<--BEGIN MESSAGE-->') self.logger.debug('{0}'.format(repr(message))) self.logger.debug('<--END MESSAGE-->') - self.leases[client_mac]["options"] = self.tlv_parse(message[240:]) + self.leases[client_mac]['options'] = self.tlv_parse(message[240:]) self.logger.debug('Parsed received options') self.logger.debug('<--BEGIN OPTIONS-->') - self.logger.debug('{0}'.format(repr(self.leases[client_mac]["options"]))) + self.logger.debug('{0}'.format(repr(self.leases[client_mac]['options']))) self.logger.debug('<--END OPTIONS-->') if not self.validate_req(client_mac): continue - type = ord(self.leases[client_mac]["options"][53][0]) # see RFC2131, page 10 + type = ord(self.leases[client_mac]['options'][53][0]) # see RFC2131, page 10 if type == 1: self.logger.debug('Received DHCPOFFER') try: self.dhcp_offer(message) except OutOfLeasesError: - self.logger.critical("Ran out of leases") + self.logger.critical('Ran out of leases') elif type == 3 and address[0] == '0.0.0.0' and not self.mode_proxy: self.logger.debug('Received DHCPACK') self.dhcp_ack(message) diff --git a/pypxe/nbd/__init__.py b/pypxe/nbd/__init__.py index 513fc2b..2c0c943 100644 --- a/pypxe/nbd/__init__.py +++ b/pypxe/nbd/__init__.py @@ -1,2 +1 @@ -__version__ = '1.0' from nbd import * diff --git a/pypxe/nbd/nbd.py b/pypxe/nbd/nbd.py index 352df96..1be9086 100644 --- a/pypxe/nbd/nbd.py +++ b/pypxe/nbd/nbd.py @@ -8,20 +8,20 @@ import writes class NBD: - def __init__(self, **serverSettings): - self.bd = serverSettings.get('blockdevice', '') - self.write = serverSettings.get('write', False) #w? - self.cow = serverSettings.get('cow', True) # COW is the safe default - self.inmem = serverSettings.get('inmem', False) - self.copytoram = serverSettings.get('copytoram', False) - self.ip = serverSettings.get('ip', '0.0.0.0') - self.port = serverSettings.get('port', 10809) - self.mode_debug = serverSettings.get('mode_debug', False) #debug mode - self.logger = serverSettings.get('logger', None) + def __init__(self, **server_settings): + self.bd = server_settings.get('block_device', '') + self.write = server_settings.get('write', False) # write? + self.cow = server_settings.get('cow', True) # COW is the safe default + self.in_mem = server_settings.get('in_mem', False) + self.copy_to_ram = server_settings.get('copy_to_ram', False) + self.ip = server_settings.get('ip', '0.0.0.0') + self.port = server_settings.get('port', 10809) + self.mode_debug = server_settings.get('mode_debug', False) # debug mode + self.logger = server_settings.get('logger', None) # setup logger if self.logger == None: - self.logger = logging.getLogger("NBD") + self.logger = logging.getLogger('NBD') handler = logging.StreamHandler() formatter = logging.Formatter('%(asctime)s %(name)s [%(levelname)s] %(message)s') handler.setFormatter(formatter) @@ -31,11 +31,11 @@ def __init__(self, **serverSettings): self.logger.setLevel(logging.DEBUG) self.logger.debug('NOTICE: NBD server started in debug mode. NBD server is using the following:') - self.logger.debug('NBD Server IP: {}'.format(self.ip)) - self.logger.debug('NBD Server Port: {}'.format(self.port)) - self.logger.debug('NBD Block Device: {}'.format(self.bd)) - self.logger.debug('NBD Block Device Writes: {}'.format(self.write)) - self.logger.debug('NBD Block Write Method: {} ({})'.format("Copy-On-Write" if self.cow else "File", "Memory" if self.inmem else "Disk")) + self.logger.debug('NBD Server IP: {0}'.format(self.ip)) + self.logger.debug('NBD Server Port: {0}'.format(self.port)) + self.logger.debug('NBD Block Device: {0}'.format(self.bd)) + self.logger.debug('NBD Block Device Writes: {0}'.format(self.write)) + self.logger.debug('NBD Block Write Method: {0} ({1})'.format("Copy-On-Write" if self.cow else 'File', 'Memory' if self.in_mem else 'Disk')) if self.mode_debug: self.logger.setLevel(logging.DEBUG) @@ -53,30 +53,29 @@ def __init__(self, **serverSettings): self.bdsize = self.openbd.tell() # go back to start self.openbd.seek(0) - if self.copytoram and self.cow: - self.logger.info('Starting copying %s to RAM', self.bd) + if self.copy_to_ram and self.cow: + self.logger.info('Starting copying {0} to RAM'.format(self.bd)) self.openbd = io.BytesIO(self.openbd.read()) - self.logger.info('Finished copying %s to RAM', self.bd) + self.logger.info('Finished copying {0} to RAM'.format(self.bd)) - - def sendreply(self, conn, addr, code, data): - '''Send a reply with magic. only used for error codes''' - reply = struct.pack("!Q", 0x3e889045565a9) - reply += struct.pack("!I", code) - reply += struct.pack("!I", len(data)) + def send_reply(self, conn, addr, code, data): + '''Send a reply with magic, only used for error codes.''' + reply = struct.pack('!Q', 0x3e889045565a9) + reply += struct.pack('!I', code) + reply += struct.pack('!I', len(data)) reply += data conn.send(reply) def handshake(self, conn, addr): - '''Initiate the connection. Server sends first.''' - # Mostly taken from https://github.com/yoe/nbd/blob/master/nbd-server.c + '''Initiate the connection, server sends first.''' + # mostly taken from https://github.com/yoe/nbd/blob/master/nbd-server.c conn.send('NBDMAGIC') # 0x49484156454F5054 conn.send('IHAVEOPT') # NBD_FLAG_FIXED_NEWSTYLE | NBD_FLAG_NO_ZEROES - conn.send(struct.pack("!H", 3)) + conn.send(struct.pack('!H', 3)) - [cflags] = struct.unpack("!I", conn.recv(4)) + [cflags] = struct.unpack('!I', conn.recv(4)) op = 0 while op != 1: # NBD_OPT_EXPORT_NAME @@ -84,47 +83,47 @@ def handshake(self, conn, addr): [op] = struct.unpack("!I", conn.recv(4)) if op != 1: # NBD_REP_ERR_UNSUP - self.sendreply(conn, addr, 2**31+1, '') + self.send_reply(conn, addr, 2 ** 31 + 1, '') - [namelen] = struct.unpack("!I", conn.recv(4)) + [namelen] = struct.unpack('!I', conn.recv(4)) name = conn.recv(namelen) if name != self.bd: conn.close() return 1 - self.logger.debug('Received request for %s from %s', name, addr) + self.logger.debug('Received request for {0} from {1}'.format(name, addr)) # size of export exportinfo = struct.pack('!Q', self.bdsize) flags = (0 if self.write else 2) # readonly? exportinfo += struct.pack('!H', flags) - exportinfo += "\x00"*(0 if (cflags&2) else 124) + exportinfo += '\x00' * (0 if (cflags & 2) else 124) conn.send(exportinfo) - def handleClient(self, conn, addr, seeklock): + def handle_client(self, conn, addr, seeklock): '''Handle all client actions, R/W/Disconnect''' ret = self.handshake(conn, addr) if ret: return # client did something wrong, so we closed them - FS = writes.write(self.cow, self.inmem)(addr, self.openbd, self.logger, seeklock) + FS = writes.write(self.cow, self.in_mem)(addr, self.openbd, self.logger, seeklock) while True: conn.recv(4) try: - [opcode, handle, offset, length] = struct.unpack("!IQQI", conn.recv(24, socket.MSG_WAITALL)) + [opcode, handle, offset, length] = struct.unpack('!IQQI', conn.recv(24, socket.MSG_WAITALL)) except struct.error: - #Client sent us something malformed, or gave up (module not loaded) + # client sent us something malformed, or gave up (module not loaded) continue if opcode not in (0, 1, 2): # NBD_REP_ERR_UNSUP - self.sendreply(conn, addr, 2**31+1, '') + self.send_reply(conn, addr, 2 ** 31 + 1, '') continue if opcode == 0: # READ data = FS.read(offset, length) - response = struct.pack("!I", 0x67446698) - response += struct.pack("!I", 0) # error - response += struct.pack("!Q", handle) + response = struct.pack('!I', 0x67446698) + response += struct.pack('!I', 0) # error + response += struct.pack('!Q', handle) conn.send(response) conn.send(data) @@ -135,34 +134,34 @@ def handleClient(self, conn, addr, seeklock): data = conn.recv(length, socket.MSG_WAITALL) FS.write(offset, data) - response = struct.pack("!I", 0x67446698) - response += struct.pack("!I", 0) # error - response += struct.pack("!Q", handle) + response = struct.pack('!I', 0x67446698) + response += struct.pack('!I', 0) # error + response += struct.pack('!Q', handle) conn.send(response) elif opcode == 2: # DISCONNECT # delete COW diff conn.close() - self.logger.debug('%s disconnected', addr) + self.logger.debug('{0} disconnected'.format(addr)) return def listen(self): - '''This method is the main loop that listens for requests''' + '''This method is the main loop that listens for requests.''' seeklock = threading.Lock() cowfiles = [] while True: try: conn, addr = self.sock.accept() - # Split off on a thread. Allows us to handle multiple clients - dispatch = threading.Thread(target = self.handleClient, args = (conn, addr, seeklock)) + # split off on a thread, allows us to handle multiple clients + dispatch = threading.Thread(target = self.handle_client, args = (conn, addr, seeklock)) # clients don't necessarily close the TCP connection - # so we use this to kill the program on Ctrl-c + # so we use this to kill the program on ctrl-c dispatch.daemon = True dispatch.start() # this is for the cleanup at the end. Will need clarifying # if MemCOW - if self.cow and not self.inmem: - cowfiles.append('PyPXE_NBD_COW_%s_%s' % addr) + if self.cow and not self.in_mem: + cowfiles.append('PyPXE_NBD_COW_{addr[0]}_{addr[1]}'.format(addr = addr)) except KeyboardInterrupt: map(os.remove, cowfiles) return diff --git a/pypxe/nbd/writes.py b/pypxe/nbd/writes.py index b1578d9..07cde16 100644 --- a/pypxe/nbd/writes.py +++ b/pypxe/nbd/writes.py @@ -3,8 +3,7 @@ class COW: def basepages(self, offset, length): # basepages is (page base addr, inter page offset, length of data in page) - # it's unlikely we'll ever need sub 4096 reads, but just in case. - # I have observed sub 4096 writes so that justifies it. + # it's unlikely we'll ever need sub 4096 reads, but just in case basepages = [] # first chunk, not necessarily at page boundary @@ -12,7 +11,7 @@ def basepages(self, offset, length): length -= 4096 - (offset % 4096) offset += 4096 - # all following FULL chunks. definate page boundary and full size + # all following FULL chunks, definate page boundary and full size while length >= 4096: basepages.append((offset, 0, 4096)) length -= 4096 @@ -28,14 +27,14 @@ def read(self, offset, length): basepages = self.basepages(offset, length) # this probably wants to be 2nd debug level - self.logger.debug('%s reading %d bytes from %s. Pages: %s', self.addr, length, hex(offset), len(basepages)) + self.logger.debug('{0} reading {1} bytes from {2}. Pages: {3}'.format(self.addr, length, hex(offset), len(basepages))) - data = "" + data = '' for major, minor, length in basepages: if major in self.pages: # major is the nth page in the file off = self.pages.index(major) - self.fh.seek(off*4096 + minor) + self.fh.seek(off * 4096 + minor) data += self.fh.read(length) else: # This is a race condition. If another thread seeks after we've @@ -43,17 +42,17 @@ def read(self, offset, length): # is wrong. # Lock is shared between all clients. Only applies to imagefd, # self.fd is unique and per client. - self.seeklock.acquire() + self.seek_lock.acquire() self.imagefd.seek(major + minor) data += self.imagefd.read(length) - self.seeklock.release() + self.seek_lock.release() return data def write(self, offset, data): basepages = self.basepages(offset, len(data)) # this probably wants to be 2nd debug level - self.logger.debug('%s writing %d bytes to %s. Pages: %s', self.addr, len(data), hex(offset), len(basepages)) + self.logger.debug('{0} writing {1} bytes to {2}. Pages: {3}'.format(self.addr, len(data), hex(offset), len(basepages))) for major, minor, length in basepages: if major in self.pages: @@ -63,83 +62,79 @@ def write(self, offset, data): data = data[length:] else: # we don't have this page, so copy it first. then add to the list - self.seeklock.acquire() + self.seek_lock.acquire() # on the page boundary self.imagefd.seek(major) cpdata = self.imagefd.read(4096) - self.seeklock.release() + self.seek_lock.release() # append to EOF self.fh.seek(0, 2) self.fh.write(cpdata) self.pages.append(major) # we've got a copy of the page now, we just need to write it off = self.pages.index(major) - self.fh.seek(off*4096 + minor) + self.fh.seek(off * 4096 + minor) self.fh.write(data[:length]) data = data[length:] class DiskCOW(COW): - def __init__(self, addr, imagefd, logger, seeklock): - # Optional argset for: - # disk diff path + def __init__(self, addr, imagefd, logger, seek_lock): + # optional argset for disk diff path self.addr = addr self.imagefd = imagefd - self.seeklock = seeklock - self.logger = logger.getChild('FS') - self.logger.debug('Copy-On-Write for %s in PyPXE_NBD_COW_%s_%s', addr, *addr) + self.seek_lock = seek_lock + self.logger = logger.get_child('FS') + self.logger.debug('Copy-On-Write for {addr} in PyPXE_NBD_COW_{addr[0]}_{addr[1]}'.format(addr = addr)) - #never want readonly cow, also definately creating file - self.fh = open('PyPXE_NBD_COW_%s_%s' % addr, 'w+b') + # never want readonly cow, also definately creating file + self.fh = open('PyPXE_NBD_COW_{addr[0]}_{addr[1]}'.format(addr = addr), 'w+b') # pages is a list of the addresses for which we have different pages - # should all be multiples of 4096. + # should all be multiples of 4096 self.pages = [] class MemCOW(COW): - def __init__(self, addr, imagefd, logger, seeklock): + def __init__(self, addr, imagefd, logger, seek_lock): self.addr = addr self.imagefd = imagefd - self.seeklock = seeklock - self.logger = logger.getChild('FS') - self.logger.debug('Copy-On-Write for %s in Memory', addr) + self.seek_lock = seek_lock + self.logger = logger.get_child('FS') + self.logger.debug('Copy-On-Write for {0} in Memory'.format(addr)) - #BytesIO looks exactly the same as a file, perfect for in memory disk + # BytesIO looks exactly the same as a file, perfect for in memory disk self.fh = io.BytesIO() # pages is a list of the addresses for which we have different pages - # should all be multiples of 4096. + # should all be multiples of 4096 self.pages = [] class RW: - def __init__(self, addr, imagefd, logger, seeklock): + def __init__(self, addr, imagefd, logger, seek_lock): self.addr = addr - self.seeklock = seeklock + self.seek_lock = seek_lock self.imagefd = imagefd - self.logger = logger.getChild('FS') - self.logger.debug('File for %s', addr) + self.logger = logger.get_child('FS') + self.logger.debug('File for {0}'.format(addr)) def read(self, offset, length): - # this probably wants to be 2nd debug level - self.logger.debug('%s reading %d bytes from %s', self.addr, length, hex(offset)) + self.logger.debug('{0} reading {1} bytes from [{2}]'.format(self.addr, length, hex(offset))) # see COW.read() for lock reason - self.seeklock.acquire() + self.seek_lock.acquire() self.imagefd.seek(offset) data = self.imagefd.read(length) - self.seeklock.release() + self.seek_lock.release() return data def write(self, offset, data): - # this probably wants to be 2nd debug level - self.logger.debug('%s writing %d bytes to %s', self.addr, len(data), hex(offset)) - self.seeklock.acquire() + self.logger.debug('{0} writing {1} bytes to {2}'.format(self.addr, len(data), hex(offset))) + self.seek_lock.acquire() self.imagefd.seek(offset) self.imagefd.write(data) - self.seeklock.release() + self.seek_lock.release() -def write(cow, inmem): - '''Class signatures are identical so we can transparently - use either.''' - if cow and inmem: +def write(cow, in_mem): + '''Class signatures are identical so we can transparently use either.''' + if cow and in_mem: return MemCOW - elif cow and not inmem: + elif cow and not in_mem: return DiskCOW else: return RW diff --git a/pypxe/tftp.py b/pypxe/tftp.py index ac4c1f4..1adff87 100644 --- a/pypxe/tftp.py +++ b/pypxe/tftp.py @@ -98,7 +98,7 @@ def parse_options(self): self.blksize = options.get('blksize', self.blksize) self.lastblock = math.ceil(self.filesize / float(self.blksize)) self.tsize = True if 'tsize' in options else False - if self.filesize > (2**16)*self.blksize: + if self.filesize > (2 ** 16) * self.blksize: self.logger.warning('Request too big, attempting transfer anyway.') self.logger.debug('Details: Filesize {0} is too big for blksize {1}.'.format(self.filesize, self.blksize)) @@ -148,7 +148,7 @@ def newRequest(self): self.send_block() return - # we got some options, so ack those first + # we got some options so ACK those first self.reply_options() def sendError(self, code = 1, message = 'File Not Found', filename = ''): @@ -182,7 +182,7 @@ def complete(self): try: self.fh.close() except AttributeError: - # we have not opened yet, or file-not-found + # we have not opened yet or file-not-found pass self.sock.close() self.dead = True From a7487a45e264b55d614558b3ba784d9b53612d6a Mon Sep 17 00:00:00 2001 From: Michael Mattioli Date: Mon, 11 May 2015 18:02:50 -0400 Subject: [PATCH 24/39] replaced string with chr(0) replaced string \x00 with chr(0) to denote a NULL byte --- pypxe/nbd/nbd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pypxe/nbd/nbd.py b/pypxe/nbd/nbd.py index 1be9086..a214229 100644 --- a/pypxe/nbd/nbd.py +++ b/pypxe/nbd/nbd.py @@ -97,7 +97,7 @@ def handshake(self, conn, addr): exportinfo = struct.pack('!Q', self.bdsize) flags = (0 if self.write else 2) # readonly? exportinfo += struct.pack('!H', flags) - exportinfo += '\x00' * (0 if (cflags & 2) else 124) + exportinfo += chr(0) * (0 if (cflags & 2) else 124) conn.send(exportinfo) def handle_client(self, conn, addr, seeklock): From e708ec57a05697e4b450a12f1e2a2587b69f32ba Mon Sep 17 00:00:00 2001 From: Michael Mattioli Date: Mon, 11 May 2015 18:07:07 -0400 Subject: [PATCH 25/39] fixed markdown table in README.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit markdown table was missing the line which denoted the table heading and wasn’t rendering properly added - to CLI options in README.md to reflect changes made in code --- README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index adec4f6..202c168 100644 --- a/README.md +++ b/README.md @@ -80,14 +80,16 @@ The following are arguments that can be passed to `pypxe-server.py` when running ##### Network Block Device Arguments |Argument|Description|Default| -|__`--nbd NBD_BLOCKDEVICE`__|Specify the block device to be served by NBD and enable NBD. This can be a disk image.|`''`| +|---|---|---| +|__`--nbd NBD_BLOCK_DEVICE`__|Specify the block device to be served by NBD and enable NBD. This can be a disk image.|`''`| |__`--nbd-write`__|Open the block device for write access. UNSAFE: Multiple clients can cause corruption|`False`| |__`--nbd-cow`__|When write is enabled, create a *volatile* file per client with their changes. Clients can write but changes are not shared or kept.|`True (Only applies if write is on)`| -|__`--nbd-cowinmem`__|Client volatile changes are stored in RAM rather than on disk. WARNING: High RAM usage (up to sizeof(block device)*clients)|`False`| -|__`--nbd-copytoram`__|Disk image is copied to RAM on start to speed up access. Changes are lost when write is used without cow.|`False`| +|__`--nbd-cow-in-mem`__|Client volatile changes are stored in RAM rather than on disk. WARNING: High RAM usage (up to sizeof(block device)*clients)|`False`| +|__`--nbd-copy-to-ram`__|Disk image is copied to RAM on start to speed up access. Changes are lost when write is used without cow.|`False`| |__`--nbd-server`__|The NBD server IP address to bind to|`0.0.0.0`| |__`--nbd-port`__|The NBD server port to bind to|`10809`| + ## Notes * `Core.iso` located in `netboot` is from the [TinyCore Project](http://distro.ibiblio.org/tinycorelinux/) and is provided as an example to network boot from using PyPXE * `chainload.kpxe` located in `netboot` is the `undionly.kpxe` from the [iPXE Project](http://ipxe.org/) From a464864728d6e0e61a5f2d0f7327bd2672da6138 Mon Sep 17 00:00:00 2001 From: Michael Mattioli Date: Mon, 11 May 2015 18:11:06 -0400 Subject: [PATCH 26/39] adjusted sample JSON configuration file for new NBD support added options to sample JSON configuration file to reflect new NBD support --- example_cfg.json | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/example_cfg.json b/example_cfg.json index 98cb347..7bf99c9 100644 --- a/example_cfg.json +++ b/example_cfg.json @@ -16,4 +16,11 @@ "USE_TFTP" : true, "USE_DHCP" : true, "DHCP_MODE_PROXY" : false, + "NBD_BLOCK_DEVICE" : "", + "NBD_WRITE" : false, + "NBD_COW" : true, + "NBD_COW_IN_MEM" : false, + "NBD_COPY_TO_RAM" : false, + "NBD_SERVER_IP" : "0.0.0.0", + "NBD_SERVER_PORT" : 10809, "MODE_DEBUG" : "" } From b13a4a3bc0e8797bae17dc47372d07db5c6f88b3 Mon Sep 17 00:00:00 2001 From: Michael Mattioli Date: Mon, 11 May 2015 18:12:18 -0400 Subject: [PATCH 27/39] renamed sample JSON DHCP leases file for uniformity renamed the sample JSON file which contains DHCP leases for uniformity with other JSON sample files --- example_dhcp_leases.json => example_leases.json | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename example_dhcp_leases.json => example_leases.json (100%) diff --git a/example_dhcp_leases.json b/example_leases.json similarity index 100% rename from example_dhcp_leases.json rename to example_leases.json From 374822d17b229959099efd122b03e0af41546724 Mon Sep 17 00:00:00 2001 From: Michael Mattioli Date: Mon, 11 May 2015 18:13:56 -0400 Subject: [PATCH 28/39] removed short CLI options from README.md removed short CLI options from README.md as they are no removed and deprecated --- README.md | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 202c168..5fd8a63 100644 --- a/README.md +++ b/README.md @@ -58,15 +58,15 @@ The following are arguments that can be passed to `pypxe-server.py` when running |Argument|Description|Default| |---|---|---| -|__`-s DHCP_SERVER_IP`__ or __`--dhcp-server-ip DHCP_SERVER_IP`__|Specify DHCP server IP address|`192.168.2.2`| -|__`-p DHCP_SERVER_PORT`__ or __`--dhcp-server-port DHCP_SERVER_PORT`__|Specify DHCP server port|`67`| -|__`-b DHCP_OFFER_BEGIN`__ or __`--dhcp-begin DHCP_OFFER_BEGIN`__|Specify DHCP lease range start|`192.168.2.100`| -|__`-e DHCP_OFFER_END`__ or __`--dhcp-end DHCP_OFFER_END`__|Specify DHCP lease range end|`192.168.2.150`| -|__`-n DHCP_SUBNET`__ or __`--dhcp-subnet DHCP_SUBNET`__|Specify DHCP subnet mask|`255.255.255.0`| -|__`-r DHCP_ROUTER`__ or __`--dhcp-router DHCP_ROUTER`__|Specify DHCP lease router|`192.168.2.1`| -|__`-d DHCP_DNS`__ or __`--dhcp-dns DHCP_DNS`__|Specify DHCP lease DNS server|`8.8.8.8`| -|__`-c DHCP_BROADCAST`__ or __`--dhcp-broadcast DHCP_BROADCAST`__|Specify DHCP broadcast address|`''`| -|__`-f DHCP_FILESERVER_IP`__ or __`--dhcp-fileserver-ip DHCP_FILESERVER_IP`__|Specify DHCP file server IP address|`192.168.2.2`| +|__`--dhcp-server-ip DHCP_SERVER_IP`__|Specify DHCP server IP address|`192.168.2.2`| +|__`--dhcp-server-port DHCP_SERVER_PORT`__|Specify DHCP server port|`67`| +|__`--dhcp-begin DHCP_OFFER_BEGIN`__|Specify DHCP lease range start|`192.168.2.100`| +|__`--dhcp-end DHCP_OFFER_END`__|Specify DHCP lease range end|`192.168.2.150`| +|__`--dhcp-subnet DHCP_SUBNET`__|Specify DHCP subnet mask|`255.255.255.0`| +| __`--dhcp-router DHCP_ROUTER`__|Specify DHCP lease router|`192.168.2.1`| +|__`--dhcp-dns DHCP_DNS`__|Specify DHCP lease DNS server|`8.8.8.8`| +|__`--dhcp-broadcast DHCP_BROADCAST`__|Specify DHCP broadcast address|`''`| +|__`--dhcp-fileserver-ip DHCP_FILESERVER_IP`__|Specify DHCP file server IP address|`192.168.2.2`| |__`--dhcp-whitelist`__|Only serve clients specified in the static lease file (`--static-config`)|`False`| @@ -74,8 +74,8 @@ The following are arguments that can be passed to `pypxe-server.py` when running |Argument|Description|Default| |---|---|---| -|__`-a NETBOOT_DIR`__ or __`--netboot-dir NETBOOT_DIR`__|Specify the local directory where network boot files will be served|`'netboot'`| -|__`-i NETBOOT_FILE`__ or __`--netboot-file NETBOOT_FILE`__|Specify the PXE boot file name|_automatically set based on what services are enabled or disabled, see [`DOCUMENTATION.md`](DOCUMENTATION.md) for further explanation_| +|__`--netboot-dir NETBOOT_DIR`__|Specify the local directory where network boot files will be served|`'netboot'`| +|__`--netboot-file NETBOOT_FILE`__|Specify the PXE boot file name|_automatically set based on what services are enabled or disabled, see [`DOCUMENTATION.md`](DOCUMENTATION.md) for further explanation_| ##### Network Block Device Arguments From 89b276b4c7c1756567645cd06533b903ee6b680f Mon Sep 17 00:00:00 2001 From: Michael Mattioli Date: Mon, 11 May 2015 18:17:23 -0400 Subject: [PATCH 29/39] added new CLI options to REAMDE.md added new CLI options for service enabling/disabling to README.md --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 5fd8a63..237b530 100644 --- a/README.md +++ b/README.md @@ -43,9 +43,13 @@ The following are arguments that can be passed to `pypxe-server.py` when running |Argument|Description|Default| |---|---|---| |__`--ipxe`__|Enable iPXE ROM|`False`| +|__`--no-ipxe`__|Disable iPXE ROM|`True`| |__`--http`__|Enable built-in HTTP server|`False`| +|__`--no-http`__|Disable built-in HTTP server|`True`| |__`--dhcp`__|Enable built-in DHCP server|`False`| |__`--dhcp-proxy`__|Enable built-in DHCP server in proxy mode (implies `--dhcp`)|`False`| +|__`--no-dhcp`__|Disable built-in DHCP server|`True`| +|__`--tftp`__|Enable built-in TFTP server which is enabled by default|`True`| |__`--no-tftp`__|Disable built-in TFTP server which is enabled by default|`False`| |__`--debug`__|Enable selected services in DEBUG mode; services are selected by passing the name in a comma separated list. **Options are: http, tftp and dhcp** _This adds a level of verbosity so that you can see what's happening in the background._|`''`| |__`--config`__|Load configuration from JSON file. (see [`example_cfg.json`](example_cfg.json))|`None`| From bb73c01f124955fa66282282096183ffe1ce6c3b Mon Sep 17 00:00:00 2001 From: Michael Mattioli Date: Mon, 11 May 2015 18:41:42 -0400 Subject: [PATCH 30/39] added debug negation description to --help MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit added description of debug negation via CLI to CLI —help --- pypxe-server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pypxe-server.py b/pypxe-server.py index 10620a7..1d84fe3 100755 --- a/pypxe-server.py +++ b/pypxe-server.py @@ -61,7 +61,7 @@ def parse_cli_arguments(): tftpexclusive.add_argument('--tftp', action = 'store_true', dest = 'USE_TFTP', help = 'Enable built-in TFTP server, by default it is enabled', default = SETTINGS['USE_TFTP']) tftpexclusive.add_argument('--no-tftp', action = 'store_false', dest = 'USE_TFTP', help = 'Disable built-in TFTP server, by default it is enabled', default = not SETTINGS['USE_TFTP']) - parser.add_argument('--debug', action = 'store', dest = 'MODE_DEBUG', help = 'Comma Seperated (http,tftp,dhcp). Adds verbosity to the selected services while they run. Use \'all\' for enabling debug on all services', default = SETTINGS['MODE_DEBUG']) + parser.add_argument('--debug', action = 'store', dest = 'MODE_DEBUG', help = 'Comma Seperated (http,tftp,dhcp). Adds verbosity to the selected services while they run. Use \'all\' for enabling debug on all services. Precede an option with \'-\' to disable debugging for that service; as an example, one can pass in the following to enable debugging for all services except the DHCP service: \'--debug all,-dhcp\'', default = SETTINGS['MODE_DEBUG']) parser.add_argument('--config', action = 'store', dest = 'JSON_CONFIG', help = 'Configure from a JSON file rather than the command line', default = '') parser.add_argument('--static-config', action = 'store', dest = 'STATIC_CONFIG', help = 'Configure leases from a json file rather than the command line', default = '') parser.add_argument('--syslog', action = 'store', dest = 'SYSLOG_SERVER', help = 'Syslog server', default = SETTINGS['SYSLOG_SERVER']) From 7e1324504943ec3b53851f1b7e585c3c2cf3c690 Mon Sep 17 00:00:00 2001 From: Michael Mattioli Date: Mon, 11 May 2015 18:45:33 -0400 Subject: [PATCH 31/39] added debug negation description to DOCUMENTATION.md added description of debug negation via CLI to the official documentation --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 237b530..0d2bf5a 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,7 @@ The following are arguments that can be passed to `pypxe-server.py` when running |__`--no-dhcp`__|Disable built-in DHCP server|`True`| |__`--tftp`__|Enable built-in TFTP server which is enabled by default|`True`| |__`--no-tftp`__|Disable built-in TFTP server which is enabled by default|`False`| -|__`--debug`__|Enable selected services in DEBUG mode; services are selected by passing the name in a comma separated list. **Options are: http, tftp and dhcp** _This adds a level of verbosity so that you can see what's happening in the background._|`''`| +|__`--debug`__|Enable selected services in DEBUG mode; services are selected by passing the name in a comma separated list. **Options are: http, tftp and dhcp**; one can also prefix an option with `-` to prevent debugging of that service; for example, the following will enable debugging for all services _except_ the DHCP service `--debug all,-dhcp`. _This mode adds a level of verbosity so that you can see what's happening in the background._|`''`| |__`--config`__|Load configuration from JSON file. (see [`example_cfg.json`](example_cfg.json))|`None`| |__`--static-config`__|Load DHCP lease configuration from JSON file. (see [`example-leases.json`](example-leases.json))|`None`| |__`--syslog`__|Specify a syslog server|`None`| From d2c402e5c57908a507faca46ad87f4494ef8487c Mon Sep 17 00:00:00 2001 From: Michael Mattioli Date: Mon, 11 May 2015 19:01:32 -0400 Subject: [PATCH 32/39] adjusted documentation and variables for DHCP service adjusted some variables for PEP8 conformity updated documentation to reflect new whitelist and static configuration support --- DOCUMENTATION.md | 2 ++ pypxe/dhcp.py | 14 +++++++------- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index 35564d6..152473a 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -104,6 +104,8 @@ The DHCP server class, __`DHCPD()`__, is constructed with the following __keywor |__`use_ipxe`__|This indicates whether or not iPXE is being used and adjusts itself accordingly.|`False`|_bool_| |__`use_http`__|This indicates whether or not the built-in HTTP server is being used and adjusts itself accordingly.|`False`|_bool_| |__`mode_proxy`__|This indicates whether or not the DHCP server should be started in ProxyDHCP mode or not.|`False`|_bool_| +|__`static_config`__|This specifies a static configuration dictionary so that it can give specific leases to specific MAC addresses.|`{}`|_dict_| +|__`whitelist`__|This indicates whether or not the DHCP server should use the static configuration dictionary as a whitelist; effectively, the DHCP server will only give out leases to those specified in the `static_config` dictionary.|`False`|_bool_| |__`mode_debug`__|This indicates whether or not the DHCP server should be started in debug mode or not.|`False`|_bool_| |__`logger`__|A [Logger](https://docs.python.org/2/library/logging.html#logger-objects) object used for logging messages, if `None` a local [StreamHandler](https://docs.python.org/2/library/logging.handlers.html#streamhandler) instance will be created.|`None`|[_Logger_](https://docs.python.org/2/library/logging.html#logger-objects)| diff --git a/pypxe/dhcp.py b/pypxe/dhcp.py index 6c5a593..7df2a2c 100644 --- a/pypxe/dhcp.py +++ b/pypxe/dhcp.py @@ -42,11 +42,11 @@ def __init__(self, **server_settings): self.ipxe = server_settings.get('use_ipxe', False) self.http = server_settings.get('use_http', False) self.mode_proxy = server_settings.get('mode_proxy', False) # ProxyDHCP mode - self.mode_debug = server_settings.get('mode_debug', False) # debug mode self.static_config = server_settings.get('static_config', dict()) self.whitelist = server_settings.get('whitelist', False) - self.magic = struct.pack('!I', 0x63825363) # magic cookie + self.mode_debug = server_settings.get('mode_debug', False) # debug mode self.logger = server_settings.get('logger', None) + self.magic = struct.pack('!I', 0x63825363) # magic cookie # setup logger if self.logger == None: @@ -210,13 +210,13 @@ def craft_options(self, opt53, client_mac): response = self.tlv_encode(53, chr(opt53)) # message type, OFFER response += self.tlv_encode(54, socket.inet_aton(self.ip)) # DHCP Server if not self.mode_proxy: - subnetmask = self.get_namespaced_static('dhcp.binding.{0}.subnet'.format(self.get_mac(client_mac)), self.subnet_mask) - response += self.tlv_encode(1, socket.inet_aton(subnetmask)) # subnet mask + subent_mask = self.get_namespaced_static('dhcp.binding.{0}.subnet'.format(self.get_mac(client_mac)), self.subnet_mask) + response += self.tlv_encode(1, socket.inet_aton(subent_mask)) # subnet mask router = self.get_namespaced_static('dhcp.binding.{0}.router'.format(self.get_mac(client_mac)), self.router) response += self.tlv_encode(3, socket.inet_aton(router)) # router - dnsserver = self.get_namespaced_static('dhcp.binding.{0}.dns'.format(self.get_mac(client_mac)), [self.dns_server]) - dnsserver = ''.join([socket.inet_aton(i) for i in dnsserver]) - response += self.tlv_encode(6, dnsserver) + dns_server = self.get_namespaced_static('dhcp.binding.{0}.dns'.format(self.get_mac(client_mac)), [self.dns_server]) + dns_server = ''.join([socket.inet_aton(i) for i in dns_server]) + response += self.tlv_encode(6, dns_server) response += self.tlv_encode(51, struct.pack('!I', 86400)) # lease time # TFTP Server OR HTTP Server; if iPXE, need both From 4f425c33a57664f1d649ce7f210cf21115612c7d Mon Sep 17 00:00:00 2001 From: PsychoMario Date: Tue, 12 May 2015 00:09:58 +0100 Subject: [PATCH 33/39] NBD documentation --- DOCUMENTATION.md | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index 152473a..d3ab250 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -42,6 +42,13 @@ We have implemented GET and HEAD, as there is no requirement for any other metho The HEAD method is used by some PXE ROMs to find the Content-Length before the GET is sent. +## NBD +NBD is similar to NFS in that it can act as a root device for Linux systems. Defined in the specification [here](https://github.com/yoe/nbd/blob/master/doc/proto.txt), NBD allows access to block devices over the network by performing read and write requests on the block device itself. + +This is different to NFS as it does not act as a filesystem, merely a single file. NBD supports read/write access along with copy-on-write support, both in memory and on disk. Read/write without copy-on-write is potentially dangerous if the file-system layer does not support multiple systems accessing it at the same time. Copy-on-write alleviates these potential problems by providing a volatile layer in which per-client changes are saved. Both the on-disk and in-memory configurations delete changes after the client disconnects, but the in-memory configuration may offer a speed increase as the changes are stored in the system RAM. + +WARNING: The use of this option can potentially consume a large amount of RAM. Up to the size of the disk image multiplied by the number of connected clients may be used. The same can be said for the on-disk configuration, where this configuration uses disk space rather than memory space. A further configuration option to store the original disk image in memory is provided to potentially allow for read/write speed up. + # PyPXE Services The PyPXE library provies the following services for the purpose of creating a Python-based PXE environment: TFTP, HTTP, and DHCP. Each service must be imorted independently as such: @@ -131,6 +138,33 @@ The HTTP server class, __`HTTPD()`__, is constructed with the following __keywor |__`mode_debug`__|This indicates whether or not the HTTP server should be started in debug mode or not.|`False`|bool| |__`logger`__|A [Logger](https://docs.python.org/2/library/logging.html#logger-objects) object used for logging messages, if `None` a local [StreamHandler](https://docs.python.org/2/library/logging.handlers.html#streamhandler) instance will be created.|`None`|[_Logger_](https://docs.python.org/2/library/logging.html#logger-objects)| +## NBD Server `pypxe.nbd` + +### Importing +The NBD service can be imported _one_ of the following two ways: +```python +from pypxe import http +``` +```python +import pypxe.nbd +``` + +### Usage +The NBD server class, __`NBD()`__, is constructed with the following __keyword arguments__: + +|Keyword Argument|Description|Default|Type| +|---|---|---|---| +|__`ip`__|This is the IP address that the NBD server will bind to.|`'0.0.0.0'` (so that it binds to all available interfaces)|_string_| +|__`port`__|This it the port that the NBD server will run on.|`10809` (default port for NBD)|_int_| +|__`block_device`__|The filename of the block device to be used as the root device|`''`|_string_| +|__`write`__|Enable write support on the block device|`False`|_bool_| +|__`cow`__|Enable copy-on-write support on the block device|`True`|_bool_| +|__`in_mem`__|Enable _in-memory_ copy-on-write support on the block device. `False` causes changes to be stored on disk|`False`|_bool_| +|__`copy_to_ram`__|Copy the disk image to RAM when the service starts|`False`|_bool_| +|__`mode_debug`__|This indicates whether or not the NBD server should be started in debug mode or not.|`False`|bool| +|__`logger`__|A [Logger](https://docs.python.org/2/library/logging.html#logger-objects) object used for logging messages, if `None` a local [StreamHandler](https://docs.python.org/2/library/logging.handlers.html#streamhandler) instance will be created.|`None`|[_Logger_](https://docs.python.org/2/library/logging.html#logger-objects)| + + ## Additional Information * The function `chr(0)` is used in multiple places throughout the servers. This denotes a `NULL` byte, or `\x00` * Python 2.6 does not include the `argparse` module, it is included in the standard library as of 2.7 and newer. The `argparse` module is required to take in command line arguments and `pypxe-server.py` will not run without it. From f59bba53702ea66040dd313f7bce6025066deabc Mon Sep 17 00:00:00 2001 From: PsychoMario Date: Tue, 12 May 2015 00:13:36 +0100 Subject: [PATCH 34/39] Fixed DOCUMENTATION table values, typos --- DOCUMENTATION.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index d3ab250..481a235 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -50,11 +50,12 @@ This is different to NFS as it does not act as a filesystem, merely a single fil WARNING: The use of this option can potentially consume a large amount of RAM. Up to the size of the disk image multiplied by the number of connected clients may be used. The same can be said for the on-disk configuration, where this configuration uses disk space rather than memory space. A further configuration option to store the original disk image in memory is provided to potentially allow for read/write speed up. # PyPXE Services -The PyPXE library provies the following services for the purpose of creating a Python-based PXE environment: TFTP, HTTP, and DHCP. Each service must be imorted independently as such: +The PyPXE library provies the following services for the purpose of creating a Python-based PXE environment: TFTP, HTTP, DHCP, and NBD. Each service must be imported independently as such: * `from pypxe import tftp` or `import pypxe.tftp` imports the TFTP service * `from pypxe import dhcp` or `import pypxe.dhcp` imports the DHCP service * `from pypxe import http` or `import pypxe.http` imports the HTTP service +* `from pypxe import nbd` or `import pypxe.nbd` imports the NBD service **See [`pypxe-server.py`](pypxe-server.py) in the root of the repo for example usage on how to call, define, and setup the services.** When running any Python script that uses these classes, it should be run as a user with root privileges as they bind to interfaces and without root privileges the services will most likely fail to bind properly. @@ -77,7 +78,7 @@ The TFTP server class, __`TFTPD()`__, is constructed with the following __keywor |__`ip`__|This is the IP address that the TFTP server will bind to.|`'0.0.0.0'` (so that it binds to all available interfaces)| _string_| |__`port`__|This it the port that the TFTP server will run on.|`69` (default port for TFTP)|_int_| |__`netboot_directory`__|This is the directory that the TFTP server will serve files from similarly to that of `tftpboot`.|`'.'` (current directory)|_string_| -|__`mode_debug`__|This indicates whether or not the TFTP server should be started in debug mode or not.|`False`|`bool`| +|__`mode_debug`__|This indicates whether or not the TFTP server should be started in debug mode or not.|`False`|_bool_| |__`logger`__|A [Logger](https://docs.python.org/2/library/logging.html#logger-objects) object used for logging messages, if `None` a local [StreamHandler](https://docs.python.org/2/library/logging.handlers.html#streamhandler) instance will be created.|`None`|[_Logger_](https://docs.python.org/2/library/logging.html#logger-objects)| |__`default_retries`__|The number of data retransmissions before dropping a connection.|`3`|_int_| |__`timeout`__|The time in seconds before re-sending an un-acknowledged data block.|`5`|_int_| @@ -135,7 +136,7 @@ The HTTP server class, __`HTTPD()`__, is constructed with the following __keywor |__`ip`__|This is the IP address that the HTTP server will bind to.|`'0.0.0.0'` (so that it binds to all available interfaces)|_string_| |__`port`__|This it the port that the HTTP server will run on.|`80` (default port for HTTP)|_int_| |__`netboot_directory`__|This is the directory that the HTTP server will serve files from similarly to that of `tftpboot`.|`'.'` (current directory)|_string_| -|__`mode_debug`__|This indicates whether or not the HTTP server should be started in debug mode or not.|`False`|bool| +|__`mode_debug`__|This indicates whether or not the HTTP server should be started in debug mode or not.|`False`|_bool_| |__`logger`__|A [Logger](https://docs.python.org/2/library/logging.html#logger-objects) object used for logging messages, if `None` a local [StreamHandler](https://docs.python.org/2/library/logging.handlers.html#streamhandler) instance will be created.|`None`|[_Logger_](https://docs.python.org/2/library/logging.html#logger-objects)| ## NBD Server `pypxe.nbd` @@ -161,7 +162,7 @@ The NBD server class, __`NBD()`__, is constructed with the following __keyword a |__`cow`__|Enable copy-on-write support on the block device|`True`|_bool_| |__`in_mem`__|Enable _in-memory_ copy-on-write support on the block device. `False` causes changes to be stored on disk|`False`|_bool_| |__`copy_to_ram`__|Copy the disk image to RAM when the service starts|`False`|_bool_| -|__`mode_debug`__|This indicates whether or not the NBD server should be started in debug mode or not.|`False`|bool| +|__`mode_debug`__|This indicates whether or not the NBD server should be started in debug mode or not.|`False`|_bool_| |__`logger`__|A [Logger](https://docs.python.org/2/library/logging.html#logger-objects) object used for logging messages, if `None` a local [StreamHandler](https://docs.python.org/2/library/logging.handlers.html#streamhandler) instance will be created.|`None`|[_Logger_](https://docs.python.org/2/library/logging.html#logger-objects)| From 8910d438e39584ec362714fcb052dcb84a815686 Mon Sep 17 00:00:00 2001 From: Michael Mattioli Date: Mon, 11 May 2015 19:22:33 -0400 Subject: [PATCH 35/39] minor styling and typos fixes in documentation adjusted some minor typos and styling fixes to the documentation --- DOCUMENTATION.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index 481a235..de3eab2 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -43,14 +43,14 @@ We have implemented GET and HEAD, as there is no requirement for any other metho The HEAD method is used by some PXE ROMs to find the Content-Length before the GET is sent. ## NBD -NBD is similar to NFS in that it can act as a root device for Linux systems. Defined in the specification [here](https://github.com/yoe/nbd/blob/master/doc/proto.txt), NBD allows access to block devices over the network by performing read and write requests on the block device itself. +NBD is similar to NFS in that it can act as a root device for Linux systems. Defined in the [specification](https://github.com/yoe/nbd/blob/master/doc/proto.txt), NBD allows access to block devices over the network by performing read and write requests on the block device itself. This is different to NFS as it does not act as a filesystem, merely a single file. NBD supports read/write access along with copy-on-write support, both in memory and on disk. Read/write without copy-on-write is potentially dangerous if the file-system layer does not support multiple systems accessing it at the same time. Copy-on-write alleviates these potential problems by providing a volatile layer in which per-client changes are saved. Both the on-disk and in-memory configurations delete changes after the client disconnects, but the in-memory configuration may offer a speed increase as the changes are stored in the system RAM. -WARNING: The use of this option can potentially consume a large amount of RAM. Up to the size of the disk image multiplied by the number of connected clients may be used. The same can be said for the on-disk configuration, where this configuration uses disk space rather than memory space. A further configuration option to store the original disk image in memory is provided to potentially allow for read/write speed up. +WARNING: The use of this option can potentially consume a large amount of RAM; up to the size of the disk image multiplied by the number of connected clients may be used. The same can be said for the on-disk configuration, where this configuration uses disk space rather than memory space. A further configuration option to store the original disk image in memory is provided to potentially allow for read/write speed up. # PyPXE Services -The PyPXE library provies the following services for the purpose of creating a Python-based PXE environment: TFTP, HTTP, DHCP, and NBD. Each service must be imported independently as such: +The PyPXE library provides the following services for the purpose of creating a Python-based PXE environment: TFTP, HTTP, DHCP, and NBD. Each service must be imported independently as such: * `from pypxe import tftp` or `import pypxe.tftp` imports the TFTP service * `from pypxe import dhcp` or `import pypxe.dhcp` imports the DHCP service @@ -94,7 +94,7 @@ from pypxe import dhcp import pypxe.dhcp ``` -###Usage +### Usage The DHCP server class, __`DHCPD()`__, is constructed with the following __keyword arguments__: |Keyword Argument|Description|Default|Type| @@ -157,11 +157,11 @@ The NBD server class, __`NBD()`__, is constructed with the following __keyword a |---|---|---|---| |__`ip`__|This is the IP address that the NBD server will bind to.|`'0.0.0.0'` (so that it binds to all available interfaces)|_string_| |__`port`__|This it the port that the NBD server will run on.|`10809` (default port for NBD)|_int_| -|__`block_device`__|The filename of the block device to be used as the root device|`''`|_string_| -|__`write`__|Enable write support on the block device|`False`|_bool_| -|__`cow`__|Enable copy-on-write support on the block device|`True`|_bool_| -|__`in_mem`__|Enable _in-memory_ copy-on-write support on the block device. `False` causes changes to be stored on disk|`False`|_bool_| -|__`copy_to_ram`__|Copy the disk image to RAM when the service starts|`False`|_bool_| +|__`block_device`__|The filename of the block device to be used as the root device.|`''`|_string_| +|__`write`__|Enable write support on the block device.|`False`|_bool_| +|__`cow`__|Enable copy-on-write support on the block device.|`True`|_bool_| +|__`in_mem`__|Enable _in-memory_ copy-on-write support on the block device. `False` causes changes to be stored on disk.|`False`|_bool_| +|__`copy_to_ram`__|Copy the disk image to RAM when the service starts.|`False`|_bool_| |__`mode_debug`__|This indicates whether or not the NBD server should be started in debug mode or not.|`False`|_bool_| |__`logger`__|A [Logger](https://docs.python.org/2/library/logging.html#logger-objects) object used for logging messages, if `None` a local [StreamHandler](https://docs.python.org/2/library/logging.handlers.html#streamhandler) instance will be created.|`None`|[_Logger_](https://docs.python.org/2/library/logging.html#logger-objects)| From d295ce0932635b1e4fd14bed37d32719702d47be Mon Sep 17 00:00:00 2001 From: Michael Mattioli Date: Mon, 11 May 2015 19:24:03 -0400 Subject: [PATCH 36/39] fix typo in DHCP class subent_mask -> subnet_mask --- pypxe/dhcp.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pypxe/dhcp.py b/pypxe/dhcp.py index 7df2a2c..b2c63ef 100644 --- a/pypxe/dhcp.py +++ b/pypxe/dhcp.py @@ -210,8 +210,8 @@ def craft_options(self, opt53, client_mac): response = self.tlv_encode(53, chr(opt53)) # message type, OFFER response += self.tlv_encode(54, socket.inet_aton(self.ip)) # DHCP Server if not self.mode_proxy: - subent_mask = self.get_namespaced_static('dhcp.binding.{0}.subnet'.format(self.get_mac(client_mac)), self.subnet_mask) - response += self.tlv_encode(1, socket.inet_aton(subent_mask)) # subnet mask + subnet_mask = self.get_namespaced_static('dhcp.binding.{0}.subnet'.format(self.get_mac(client_mac)), self.subnet_mask) + response += self.tlv_encode(1, socket.inet_aton(subnet_mask)) # subnet mask router = self.get_namespaced_static('dhcp.binding.{0}.router'.format(self.get_mac(client_mac)), self.router) response += self.tlv_encode(3, socket.inet_aton(router)) # router dns_server = self.get_namespaced_static('dhcp.binding.{0}.dns'.format(self.get_mac(client_mac)), [self.dns_server]) From e9782b4a2db4be224909b945f174932f7895b09d Mon Sep 17 00:00:00 2001 From: PsychoMario Date: Tue, 12 May 2015 17:24:06 +0100 Subject: [PATCH 37/39] Fixed argument change --- pypxe-server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pypxe-server.py b/pypxe-server.py index 1d84fe3..f3c6e5c 100755 --- a/pypxe-server.py +++ b/pypxe-server.py @@ -178,7 +178,7 @@ def do_debug(service): if args.NBD_WRITE and not args.NBD_COW: sys_logger.warning('NBD Write enabled but copy-on-write is not. Multiple clients may cause corruption') - if args.NBD_COWINMEM or args.NBD_COPYTORAM: + if args.NBD_COW_IN_MEM or args.NBD_COPY_TO_RAM: sys_logger.warning('NBD cowinmem and copytoram can cause high RAM usage') #serve all files from one directory From 3d7fa3ffbc4ef816934951368ba311dd0ece616a Mon Sep 17 00:00:00 2001 From: PsychoMario Date: Tue, 12 May 2015 17:29:07 +0100 Subject: [PATCH 38/39] fixed nbd writes.py child logging --- pypxe/nbd/writes.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pypxe/nbd/writes.py b/pypxe/nbd/writes.py index 07cde16..30cd787 100644 --- a/pypxe/nbd/writes.py +++ b/pypxe/nbd/writes.py @@ -83,7 +83,7 @@ def __init__(self, addr, imagefd, logger, seek_lock): self.addr = addr self.imagefd = imagefd self.seek_lock = seek_lock - self.logger = logger.get_child('FS') + self.logger = logger.getChild('FS') self.logger.debug('Copy-On-Write for {addr} in PyPXE_NBD_COW_{addr[0]}_{addr[1]}'.format(addr = addr)) # never want readonly cow, also definately creating file @@ -97,7 +97,7 @@ def __init__(self, addr, imagefd, logger, seek_lock): self.addr = addr self.imagefd = imagefd self.seek_lock = seek_lock - self.logger = logger.get_child('FS') + self.logger = logger.getChild('FS') self.logger.debug('Copy-On-Write for {0} in Memory'.format(addr)) # BytesIO looks exactly the same as a file, perfect for in memory disk @@ -111,7 +111,7 @@ def __init__(self, addr, imagefd, logger, seek_lock): self.addr = addr self.seek_lock = seek_lock self.imagefd = imagefd - self.logger = logger.get_child('FS') + self.logger = logger.getChild('FS') self.logger.debug('File for {0}'.format(addr)) def read(self, offset, length): From 6b3107cc3dd4c1530529245a35612c8d8f244abb Mon Sep 17 00:00:00 2001 From: Michael Mattioli Date: Tue, 12 May 2015 13:42:47 -0400 Subject: [PATCH 39/39] remove redundant service labels from debug messages remove labels from debug messages denoting which service they belong to, this is redundant with the use of the logger --- pypxe/dhcp.py | 18 +++++++++--------- pypxe/http.py | 6 +++--- pypxe/nbd/nbd.py | 10 +++++----- pypxe/tftp.py | 6 +++--- 4 files changed, 20 insertions(+), 20 deletions(-) diff --git a/pypxe/dhcp.py b/pypxe/dhcp.py index b2c63ef..46441de 100644 --- a/pypxe/dhcp.py +++ b/pypxe/dhcp.py @@ -71,18 +71,18 @@ def __init__(self, **server_settings): # debug info for ProxyDHCP mode if not self.mode_proxy: - self.logger.debug('DHCP Lease Range: {0} - {1}'.format(self.offer_from, self.offer_to)) - self.logger.debug('DHCP Subnet Mask: {0}'.format(self.subnet_mask)) - self.logger.debug('DHCP Router: {0}'.format(self.router)) - self.logger.debug('DHCP DNS Server: {0}'.format(self.dns_server)) - self.logger.debug('DHCP Broadcast Address: {0}'.format(self.broadcast)) + self.logger.debug('Lease Range: {0} - {1}'.format(self.offer_from, self.offer_to)) + self.logger.debug('Subnet Mask: {0}'.format(self.subnet_mask)) + self.logger.debug('Router: {0}'.format(self.router)) + self.logger.debug('DNS Server: {0}'.format(self.dns_server)) + self.logger.debug('Broadcast Address: {0}'.format(self.broadcast)) if self.static_config: - self.logger.debug('DHCP Using Static Leasing') - self.logger.debug('DHCP Using Static Leasing Whitelist: {0}'.format(self.whitelist)) + self.logger.debug('Using Static Leasing') + self.logger.debug('Using Static Leasing Whitelist: {0}'.format(self.whitelist)) - self.logger.debug('DHCP File Server IP: {0}'.format(self.file_server)) - self.logger.debug('DHCP File Name: {0}'.format(self.file_name)) + self.logger.debug('File Server IP: {0}'.format(self.file_server)) + self.logger.debug('File Name: {0}'.format(self.file_name)) self.logger.debug('ProxyDHCP Mode: {0}'.format(self.mode_proxy)) self.logger.debug('Using iPXE: {0}'.format(self.ipxe)) self.logger.debug('Using HTTP Server: {0}'.format(self.http)) diff --git a/pypxe/http.py b/pypxe/http.py index d1f97fc..2ef325a 100644 --- a/pypxe/http.py +++ b/pypxe/http.py @@ -45,9 +45,9 @@ def __init__(self, **server_settings): os.chroot ('.') self.logger.debug('NOTICE: HTTP server started in debug mode. HTTP server is using the following:') - self.logger.debug('HTTP Server IP: {0}'.format(self.ip)) - self.logger.debug('HTTP Server Port: {0}'.format(self.port)) - self.logger.debug('HTTP Network Boot Directory: {0}'.format(self.netboot_directory)) + self.logger.debug('Server IP: {0}'.format(self.ip)) + self.logger.debug('Server Port: {0}'.format(self.port)) + self.logger.debug('Network Boot Directory: {0}'.format(self.netboot_directory)) def handle_request(self, connection, addr): '''This method handles HTTP request.''' diff --git a/pypxe/nbd/nbd.py b/pypxe/nbd/nbd.py index a214229..8a1ad53 100644 --- a/pypxe/nbd/nbd.py +++ b/pypxe/nbd/nbd.py @@ -31,11 +31,11 @@ def __init__(self, **server_settings): self.logger.setLevel(logging.DEBUG) self.logger.debug('NOTICE: NBD server started in debug mode. NBD server is using the following:') - self.logger.debug('NBD Server IP: {0}'.format(self.ip)) - self.logger.debug('NBD Server Port: {0}'.format(self.port)) - self.logger.debug('NBD Block Device: {0}'.format(self.bd)) - self.logger.debug('NBD Block Device Writes: {0}'.format(self.write)) - self.logger.debug('NBD Block Write Method: {0} ({1})'.format("Copy-On-Write" if self.cow else 'File', 'Memory' if self.in_mem else 'Disk')) + self.logger.debug('Server IP: {0}'.format(self.ip)) + self.logger.debug('Server Port: {0}'.format(self.port)) + self.logger.debug('Block Device: {0}'.format(self.bd)) + self.logger.debug('Block Device Writes: {0}'.format(self.write)) + self.logger.debug('Block Write Method: {0} ({1})'.format("Copy-On-Write" if self.cow else 'File', 'Memory' if self.in_mem else 'Disk')) if self.mode_debug: self.logger.setLevel(logging.DEBUG) diff --git a/pypxe/tftp.py b/pypxe/tftp.py index 1adff87..d436da5 100644 --- a/pypxe/tftp.py +++ b/pypxe/tftp.py @@ -238,9 +238,9 @@ def __init__(self, **server_settings): self.logger.setLevel(logging.DEBUG) self.logger.debug('NOTICE: TFTP server started in debug mode. TFTP server is using the following:') - self.logger.debug('TFTP Server IP: {0}'.format(self.ip)) - self.logger.debug('TFTP Server Port: {0}'.format(self.port)) - self.logger.debug('TFTP Network Boot Directory: {0}'.format(self.netbook_directory)) + self.logger.debug('Server IP: {0}'.format(self.ip)) + self.logger.debug('Server Port: {0}'.format(self.port)) + self.logger.debug('Network Boot Directory: {0}'.format(self.netbook_directory)) self.ongoing = []