pike.core

1 # 2 # Copyright (c) 2013, EMC Corporation 3 # All rights reserved. 4 # 5 # Redistribution and use in source and binary forms, with or without 6 # modification, are permitted provided that the following conditions are met: 7 # 8 # 1. Redistributions of source code must retain the above copyright notice, 9 # this list of conditions and the following disclaimer. 10 # 2. Redistributions in binary form must reproduce the above copyright notice, 11 # this list of conditions and the following disclaimer in the documentation 12 # and/or other materials provided with the distribution. 13 # 14 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 15 # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 18 # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 19 # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 20 # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 21 # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 22 # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 23 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 24 # POSSIBILITY OF SUCH DAMAGE. 25 # 26 # Module Name: 27 # 28 # core.py 29 # 30 # Abstract: 31 # 32 # Core API 33 # 34 # Authors: Brian Koropoff (brian.koropoff@emc.com) 35 # 36 37 """ 38 Core Pike infrastructure 39 """ 40 41 import array 42 import struct 43 import inspect

44 45 -class BufferOverrun(Exception):

46 """Buffer overrun exception""" 47 pass

48

49 -class Cursor(object):

50 """ 51 Byte array cursor 52 53 Represents a position within a byte array. 54 55 encode_* operations write data at the current position 56 and advance it by the number of bytes written. 57 58 decode_* operations read data at the current position, 59 advance it by the number of bytes read, and return 60 the result. 61 62 Cursors support in-place addition and subtraction (+=,-=) of 63 integer values to manually adjust position, as well 64 as ordinary addition and subtraction, which return a new 65 cursor without modifiying the original. 66 67 Ordinary subtraction of two cursors will yield the 68 difference between their positions as an integer. 69 ==, !=, <, <=, >, and >= operators will also work to 70 compare positions. The results will only be meaningful 71 for cursors referencing the same underlying array. 72 73 For a given cursor, cursor.hole.encode_* will perform 74 the same operation as cursor.encode_*, but will return 75 a hole object. Calling this hole object with the same 76 type of arguments as the original encode method will 77 overwrite the original value. This mechanism is useful 78 for backpatching values into fields that aren't known until 79 later, such as internal packet lengths and offsets or 80 checksum fields. For example:: 81 82 hole = cursor.hole.encode_uint32le(0) 83 # Encode rest of packet 84 ... 85 # Overwrite value with calculated checksum 86 hole(sum) 87 88 Cursors support slicing to extract sections 89 of the underlying array. For example:: 90 91 # Extract array slice between cur1 and cur2 92 subarray = cur1[:cur2] 93 94 Cursors also support establishing boundaries outside of which 95 decoding will raise exceptions:: 96 97 with cur.bounded(startcur, endcur): 98 # Within this block, attempts to decode data outside of 99 # the range starting with startcur (inclusive) and ending 100 # with endcur (exclusive) will raise BufferOverrun(). 101 # If the start and end paremeters are numbers rather than 102 # other cursors, they will be taken to be relative to 103 # cur itself. 104 ... 105 106 @ivar array: Array referenced by cursor 107 @ivar offset: Offset within the array 108 @ivar bounds: Pair of lower and upper bound on offset 109 """ 110

111 - def __init__(self, arr, offset, bounds=(None,None)):

112 """ 113 Create a L{Cursor} for the given array 114 at the given offset. 115 116 @type arr: array.array('B', ...) 117 @param arr: The array 118 @type offset: number 119 @param offset: The offset from the start of the array 120 @param bounds: A pair of a lower and upper bound on valid offsets 121 """ 122 123 self.array = arr 124 self.offset = offset 125 self.bounds = bounds 126 self.hole = Cursor.Hole(self)

127

128 - def __eq__(self, o):

129 return self.array is o.array and self.offset == o.offset

130

131 - def __ne__(self, o):

132 return not (self == o)

133

134 - def __lt__(self, o):

135 assert self.array is o.array 136 return self.offset < o.offset

137

138 - def __gt__(self, o):

139 assert self.array is o.array 140 return self.offset > o.offset

141

142 - def __le__(self, o):

143 assert self.array is o.array 144 return self.offset <= o.offset

145

146 - def __ge__(self, o):

147 assert self.array is o.array 148 return self.offset >= o.offset

149

150 - def __add__(self, o):

151 return Cursor(self.array, self.offset + o, self.bounds)

152

153 - def __sub__(self, o):

154 if isinstance(o, Cursor): 155 assert self.array is o.array 156 return self.offset - o.offset 157 else: 158 return Cursor(self.array, self.offset - o, self.bounds)

159

160 - def __iadd__(self, o):

161 self.offset += o 162 return self

163

164 - def __isub__(self, o):

165 self.offset -= o 166 return self

167

168 - def _getindex(self, ind):

169 if ind is None: 170 return None 171 elif isinstance(ind, Cursor): 172 assert self.array is ind.array 173 return ind.offset 174 else: 175 return self.offset + ind

176

177 - def __getitem__(self, index):

178 if isinstance(index, slice): 179 start = self._getindex(index.start if index.start else 0) 180 stop = self._getindex(index.stop) 181 step = index.step 182 183 self._check_bounds(start, stop) 184 return self.array.__getitem__(slice(start,stop,step)) 185 else: 186 self._check_bounds(index, index+1) 187 return self.array.__getitem__(index)

188

189 - def __repr__(self):

190 return 'Cursor(' + object.__repr__(self.array) + ',' + repr(self.offset) + ',' + repr(self.bounds) + ')'

191

192 - def copy(self):

193 """ Create copy of cursor. """ 194 return Cursor(self.array, self.offset, self.bounds)

195

196 - def _expand_to(self, size):

197 cur_size = len(self.array) 198 if (size > cur_size): 199 self.array.extend([0]*(size - cur_size))

200

201 - def encode_bytes(self, val):

202 """ Encode bytes. Accepts byte arrays, strings, and integer lists.""" 203 size = len(val) 204 self._expand_to(self.offset + size) 205 self.array[self.offset:self.offset + size] = array.array('B',val) 206 self.offset += size

207

208 - def encode_struct(self, fmt, *args):

209 size = struct.calcsize(fmt) 210 self._expand_to(self.offset + size) 211 struct.pack_into(fmt, self.array, self.offset, *args) 212 self.offset += size

213

214 - def encode_uint8be(self, val):

215 self.encode_struct('>B', val)

216

217 - def encode_uint16be(self, val):

218 self.encode_struct('>H', val)

219

220 - def encode_uint32be(self, val):

221 self.encode_struct('>L', val)

222

223 - def encode_uint64be(self, val):

224 self.encode_struct('>Q', val)

225

226 - def encode_uint8le(self, val):

227 self.encode_struct('<B', val)

228

229 - def encode_uint16le(self, val):

230 self.encode_struct('<H', val)

231

232 - def encode_uint32le(self, val):

233 self.encode_struct('<L', val)

234

235 - def encode_uint64le(self, val):

236 self.encode_struct('<Q', val)

237

238 - def encode_int64le(self, val):

239 self.encode_struct('<q', val)

240

241 - def encode_utf16le(self, val):

242 self.encode_bytes(unicode(val).encode('utf-16le'))

243

244 - def trunc(self):

245 self._expand_to(self.offset) 246 del self.array[self.offset:]

247

248 - def _check_bounds(self, start, end):

249 lower = self.bounds[0] if self.bounds[0] is not None else 0 250 upper = self.bounds[1] if self.bounds[1] is not None else len(self.array) 251 252 if start < lower or end > upper: 253 raise BufferOverrun()

254

255 - def decode_bytes(self, size):

256 self._check_bounds(self.offset, self.offset + size) 257 result = self.array[self.offset:self.offset+size] 258 self.offset += size 259 return result

260

261 - def decode_struct(self, fmt):

262 size = struct.calcsize(fmt) 263 self._check_bounds(self.offset, self.offset + size) 264 result = struct.unpack_from(fmt, self.array, self.offset) 265 self.offset += size 266 return result

267

268 - def decode_uint8be(self):

269 return self.decode_struct('>B')[0]

270

271 - def decode_uint16be(self):

272 return self.decode_struct('>H')[0]

273

274 - def decode_uint32be(self):

275 return self.decode_struct('>L')[0]

276

277 - def decode_uint64be(self):

278 return self.decode_struct('>Q')[0]

279

280 - def decode_uint8le(self):

281 return self.decode_struct('<B')[0]

282

283 - def decode_uint16le(self):

284 return self.decode_struct('<H')[0]

285

286 - def decode_uint32le(self):

287 return self.decode_struct('<L')[0]

288

289 - def decode_int32le(self):

290 return self.decode_struct('<l')[0]

291

292 - def decode_uint64le(self):

293 return self.decode_struct('<Q')[0]

294

295 - def decode_int64le(self):

296 return self.decode_struct('<q')[0]

297

298 - def decode_utf16le(self, size):

299 return self.decode_bytes(size).tostring().decode('utf-16le')

300

301 - def align(self, base, val):

302 assert self.array is base.array 303 rem = (self.offset - base.offset) % val 304 if rem != 0: 305 self.offset += val - rem

306

307 - def seekto(self, o, lowerbound = None, upperbound = None):

308 assert self.array is o.array 309 if (lowerbound is not None and o < lowerbound) or \ 310 (upperbound is not None and o > upperbound): 311 raise BufferOverrun() 312 self.offset = o.offset

313

314 - def advanceto(self, o, bound = None):

315 self.seekto(o, self, bound)

316

317 - def reverseto(self, o, bound = None):

318 self.seekto(o, bound, self)

319 320 @property

321 - def lowerbound(self):

322 lower = self.bounds[0] if self.bounds[0] is not None else 0 323 return Cursor(self.array, lower, self.bounds)

324 325 @property

326 - def upperbound(self):

327 upper = self.bounds[1] if self.bounds[1] is not None else len(self.array) 328 return Cursor(self.array, upper, self.bounds)

329

330 - def bounded(self, lower, upper):

331 # Allow cursors to be used as bounds (the preferred idiom) 332 if isinstance(lower, Cursor): 333 assert self.array is lower.array 334 lower = lower.offset 335 else: 336 lower = cur.offset + lower 337 if isinstance(upper, Cursor): 338 assert self.array is upper.array 339 upper = upper.offset 340 else: 341 upper = cur.offset + upper 342 343 # Don't let new bounds escape current bounds 344 self._check_bounds(lower, upper) 345 346 return Cursor.Bounds(self, lower, upper)

347

348 - class Hole(object):

349 - def __init__(self, cur):

350 self.cur = cur

351 - def __getattr__(self, attr):

352 if hasattr(self.cur.__class__, attr): 353 f = getattr(self.cur.__class__, attr) 354 if inspect.ismethod(f): 355 copy = self.cur.copy() 356 def f2(*args, **kwargs): 357 offset = copy.offset 358 f(copy,*args,**kwargs) 359 copy.offset = offset

360 361 def f1(*args, **kwargs): 362 f(self.cur,*args,**kwargs) 363 return f2

364 365 return f1 366 else: 367 raise AttributeError 368 else: 369 raise AttributeError

370

371 - class Bounds(object):

372 - def __init__(self, cur, lower, upper):

373 self.cur = cur 374 self.bounds = (lower,upper)

375

376 - def __enter__(self):

377 self.oldbounds = self.cur.bounds 378 self.cur.bounds = self.bounds 379 return self

380

381 - def __exit__(self, exc_type, exc_value, traceback):

382 self.cur.bounds = self.oldbounds

383

384 -class BadPacket(Exception):

385 pass

386

387 -class Frame(object):

388 field_blacklist = ['fields','parent','start','end'] 389

390 - def __init__(self, parent, context=None):

391 object.__setattr__(self, 'fields', []) 392 self.parent = parent 393 self._context = context

394

395 - def __len__(self):

396 return len(self.children)

397

398 - def __getitem__(self, key):

399 return self.children[key]

400

401 - def __iter__(self):

402 return self.children.__iter__()

403

404 - def __setattr__(self, name, value):

405 if not name.startswith('_') and \ 406 name not in self.fields and \ 407 name not in self.field_blacklist: 408 self.fields.append(name) 409 object.__setattr__(self, name, value)

410

411 - def __str__(self):

412 return self._str(1)

413

414 - def _value_str(self, value):

415 if isinstance(value, array.array) and value.typecode == 'B': 416 return '0x' + ''.join(map(lambda b:'%.2x'%b,value)) 417 else: 418 return str(value)

419

420 - def _str(self, indent):

421 res = self.__class__.__name__ 422 for field in self.fields: 423 value = getattr(self, field) 424 if value is not None: 425 if isinstance(value, Frame): 426 valstr = value._str(indent + 1) 427 else: 428 valstr = self._value_str(value) 429 res += "\n" + " " * indent + field + ": " + valstr 430 for child in self.children: 431 valstr = child._str(indent + 1) 432 res += "\n" + " " * indent + valstr 433 return res

434

435 - def _encode_pre(self, cur):

436 self.start = cur.copy()

437

438 - def _encode_post(self, cur):

439 self.end = cur.copy()

440

441 - def _decode_pre(self, cur):

442 self.start = cur.copy()

443

444 - def _decode_post(self, cur):

445 self.end = cur.copy()

446 447 @property

448 - def context(self):

449 if self._context is not None: 450 return self._context 451 elif self.parent is not None: 452 return self.parent.context 453 else: 454 return None

455 456 @property

457 - def children(self):

458 return self._children() if hasattr(self, '_children') else []

459

460 - def encode(self, cur):

461 self._encode_pre(cur) 462 self._encode(cur) 463 self._encode_post(cur)

464

465 - def decode(self, cur):

466 self._decode_pre(cur) 467 self._decode(cur) 468 self._decode_post(cur)

469

470 - def serialize(self):

471 self.buf = array.array('B') 472 cursor = Cursor(self.buf, 0) 473 self.encode(cursor) 474 return self.buf

475

476 - def parse(self, arr):

477 cursor = Cursor(arr, 0) 478 self.decode(cursor) 479 self.buf = arr

480

481 - def next_sibling(self):

482 children = self.parent.children 483 index = children.index(self) + 1 484 return children[index] if index < len(children) else None

485

486 - def prev_sibling(self):

487 children = self.parent.children 488 index = children.index(self) - 1 489 return children[index] if index >= 0 else None

490

491 - def is_last_child(self):

492 children = self.parent.children 493 return children.index(self) == len(children) - 1

494

495 - class __metaclass__(type):

496 - def __new__(mcs, name, bases, dict):

497 # Inherit _register from bases 498 dict['_register'] = [] 499 for base in bases: 500 if hasattr(base, '_register'): 501 dict['_register'] += base._register 502 503 # Inherit field_blacklist from bases 504 if 'field_blacklist' in dict: 505 for base in bases: 506 if hasattr(base,'field_blacklist'): 507 dict['field_blacklist'] += base.field_blacklist 508 509 result = type.__new__(mcs, name, bases, dict) 510 511 # Register class in appropriate tables 512 for (table,keyattrs) in result._register: 513 if all(hasattr(result, a) for a in keyattrs): 514 key = [getattr(result, a) for a in keyattrs] 515 if len(key) == 1: 516 key = key[0] 517 else: 518 key = tuple(key) 519 table[key] = result 520 521 return result

522

523 -class Register(object):

524 - def __init__(self, table, *keyattrs):

525 self.table = table 526 self.keyattrs = keyattrs

527

528 - def __call__(self, cls):

529 assert issubclass(cls, Frame) 530 cls._register.append((self.table,self.keyattrs)) 531 return cls

532

533 -class Enum(long):

534 """ 535 Enumeration abstract base 536 537 An Enum subclasses long in order to ensure that instances 538 are a member of a well-defined enumeration of values, 539 provide introspection into the allowed values and their names, 540 and provide symbolic string forms of values. 541 542 You should generally subclass one of L{ValueEnum} or L{FlagEnum}. 543 """ 544 545 @classmethod

546 - def items(cls):

547 """ 548 Returns a list of (name,value) pairs for allowed enumeration values. 549 """ 550 return cls._nametoval.iteritems()

551 552 @classmethod

553 - def names(cls):

554 """ 555 Returns a list of names of allowed enumeration values. 556 """ 557 return [name for (name,value) in cls.items()]

558 559 @classmethod

560 - def values(cls):

561 """ 562 Returns a list of allowed enumeration values. 563 """ 564 return [value for (name,value) in cls.items()]

565 566 @classmethod

567 - def import_items(cls, dictionary):

568 """ 569 Import enumeration values into dictionary 570 571 This method is intended to allow importing enumeration 572 values from an L{Enum} subclass into the root of a module, 573 and should be invoked as:: 574 575 SomeEnumClass.import_items(globals()) 576 """ 577 dictionary.update((name,cls(value)) for (name,value) in cls.items())

578 579 @classmethod

580 - def validate(cls, value):

581 """ 582 Validate value as valid for enumeration. 583 584 This must be implemented in subclasses of L{Enum} 585 (but not L{ValueEnum} or L{FlagEnum}, which provide it). 586 It should raise a ValueError on failure. 587 """ 588 raise NotImplementedError()

589

590 - def __new__(cls, value=0):

591 """ 592 Constructor. 593 594 Creates a new Enum instance from an ordinary number, 595 validating that is is valid for the particular 596 enumeration. 597 """ 598 cls.validate(value) 599 return super(Enum, cls).__new__(cls, value)

600

601 - def __repr__(self):

602 # Just return string form 603 return str(self)

604

605 - class __metaclass__(type):

606 - def __new__(mcs, cname, bases, idict):

607 nametoval = {} 608 valtoname = {} 609 misc = {} 610 611 for (name,val) in idict.iteritems(): 612 if name[0].isupper(): 613 nametoval[name] = val 614 valtoname[val] = name 615 else: 616 misc[name] = val 617 618 cls = type.__new__(mcs, cname, bases, misc) 619 cls._nametoval = nametoval 620 cls._valtoname = valtoname 621 622 return cls

623

624 - def __getattribute__(cls, name):

625 nametoval = type.__getattribute__(cls, '_nametoval') 626 if name in nametoval: 627 return cls(nametoval[name]) 628 else: 629 return type.__getattribute__(cls, name)

630

631 -class ValueEnum(Enum):

632 """ 633 Value Enumeration 634 635 Subclass this class to define enumerations of values. 636 For example:: 637 638 class SomeEnumClass(Enum): 639 FOO = 0 640 BAR = 1 641 BAZ = 2 642 643 Accessing SomeEnumClass.FOO will actually return a SomeEnumClass instance. 644 Instances will return their symbolic names when str() is used. 645 """ 646 647 permissive = False 648 649 @classmethod

650 - def validate(cls, value):

651 if not cls.permissive and value not in cls._valtoname: 652 raise ValueError("Invalid %s: %x" % (cls.__name__, value))

653

654 - def __str__(self):

655 if self in self.__class__._valtoname: 656 return self.__class__._valtoname[self] 657 else: 658 return hex(self)

659

660 -class FlagEnum(Enum):

661 """ 662 Flag Enumeration 663 664 Subclass this class to define enumerations of flags. 665 For example:: 666 667 class SomeFlagEnumClass(Enum): 668 FOO = 0x1 669 BAR = 0x2 670 BAZ = 0x4 671 672 Accessing SomeFlagEnumClass.FOO will actually return a 673 SomeFlagEnumClass instance. Instances may be bitwise-ored 674 together. Instances will return the symbolic names of all 675 set flags joined by ' | ' when str() is used. 676 """ 677 678 @classmethod

679 - def validate(cls, value):

680 remaining = value 681 for flag in cls.values(): 682 if flag & remaining == flag: 683 remaining &= ~flag 684 685 if remaining != 0: 686 raise ValueError("Invalid %s: 0x%x (remainder 0x%x)" % (cls.__name__, value, remaining))

687

688 - def __str__(self):

689 names = [name for (name,flag) in self.items() 690 if (flag != 0 and flag & self == flag) or 691 (self == 0 and flag == 0)] 692 693 return ' | '.join(names) if len(names) else '0'

694

695 - def __or__(self, o):

696 return self.__class__(super(FlagEnum,self).__or__(o))

697

698 - def __and__(self, o):

699 return self.__class__(super(FlagEnum,self).__and__(o))

700

701 -class Let(object):

702 """ 703 A Let allows for temporarily storing passed arguments in the _settings 704 member of a target object for the duration of the contextmanager's scope 705 706 Implementation on a factory class 707 708 class MyThingFactory(object): 709 def __init__(self): 710 self._settings = {} 711 def generate_thing(self): 712 # do something here 713 new_obj = Thing() 714 for k,v in self._settings.iteritems(): 715 setattr(new_obj, k, v) 716 return new_obj 717 def let(self, **kwds): 718 return Let(self, kwds) 719 720 Calling code that wants to temporarily customize the Thing objects that are 721 returned by the factory: 722 723 fac = MyThingFactory() 724 with fac.let(this="that", foo="bar"): 725 a_thing = fac.generate_thing() 726 self.assertEqual(a_thing.this, "that") 727 """

728 - def __init__(self, target, settings_dict):

729 self.target = target 730 self.settings_dict = settings_dict 731 if not hasattr(target, "_settings"): 732 target._settings = {}

733

734 - def __enter__(self):

735 self.backup = dict(self.target._settings) 736 self.target._settings.update(self.settings_dict) 737 return self

738

739 - def __exit__(self, exc_type, exc_val, exc_tb):

740 self.target._settings = self.backup

741

Source Code for Module pike.core