Changes required for pykolab to work with AD

Summary: These changes basically are to remove referrals from the ldapsearch results. The change is cache sqlite DB schema is required to allow objectGUID AD attribute to work as unique attribute to track LDAP objects.

Reviewers: vanmeeuwen, machniak, mollekopf

Reviewed By: machniak

Subscribers: #pykolab_developers

Tags: #pykolab

Differential Revision: https://git.kolab.org/D720

5 files changed, 62 insertions, 27 deletions

diff --git a/.arclint b/.arclint
index a5496f4..56cd4ba 100644
--- a/.arclint
+++ b/.arclint
@@ -24,6 +24,7 @@
                 "E251": "disabled",
                 "E261": "disabled",
                 "E265": "disabled",
+                "E266": "disabled",
                 "E302": "disabled",
                 "E303": "disabled",
                 "E402": "disabled",
diff --git a/pykolab/auth/ldap/__init__.py b/pykolab/auth/ldap/__init__.py
index 0c440da..8400bfb 100644
--- a/pykolab/auth/ldap/__init__.py
+++ b/pykolab/auth/ldap/__init__.py
@@ -502,10 +502,12 @@ class LDAP(pykolab.base.Base):
         else:
             base_dn = config_base_dn
 
+        _filter = "(%s=%s)" % (unique_attribute, ldap.filter.escape_filter_chars(entry_id))
+
         _search = self.ldap.search_ext(
                 base_dn,
                 ldap.SCOPE_SUBTREE,
-                '(%s=%s)' % (unique_attribute, entry_id),
+                _filter,
                 ['entrydn']
             )
 
@@ -718,7 +720,7 @@ class LDAP(pykolab.base.Base):
             __filter_prefix = "(&"
             __filter_suffix = "(!(%s=%s)))" % (
                     self.config_get('unique_attribute'),
-                    exclude_entry_id
+                    ldap.filter.escape_filter_chars(exclude_entry_id)
                 )
 
         else:
@@ -794,7 +796,7 @@ class LDAP(pykolab.base.Base):
             __filter_prefix = "(&"
             __filter_suffix = "(!(%s=%s)))" % (
                     self.config_get('unique_attribute'),
-                    exclude_entry_id
+                    ldap.filter.escape_filter_chars(exclude_entry_id)
                 )
 
         else:
@@ -845,11 +847,8 @@ class LDAP(pykolab.base.Base):
                 attrsonly=True
             )
 
-        _entry_dns = []
-
-        for _result in _results:
-            (_entry_id, _entry_attrs) = _result
-            _entry_dns.append(_entry_id)
+        # Remove referrals
+        _entry_dns = [_e for _e in _results if _e[0] is not None]
 
         return _entry_dns
 
@@ -1232,7 +1231,7 @@ class LDAP(pykolab.base.Base):
         else:
             base_dn = config_base_dn
 
-        return self._search(
+        _results = self._search(
                 base_dn,
                 filterstr=_filter,
                 attrlist=[
@@ -1241,6 +1240,11 @@ class LDAP(pykolab.base.Base):
                 override_search='_regular_search'
             )
 
+        # Remove referrals
+        _entry_dns = [_e for _e in _results if _e[0] is not None]
+
+        return _entry_dns
+
     def set_entry_attribute(self, entry_id, attribute, value):
         log.debug(_("Setting entry attribute %r to %r for %r") % (attribute, value, entry_id), level=8)
         self.set_entry_attributes(entry_id, { attribute: value })
@@ -1296,7 +1300,7 @@ class LDAP(pykolab.base.Base):
             if not conf.resync:
                 modified_after = self.get_latest_sync_timestamp()
             else:
-                modifytimestamp_format = conf.get_raw('ldap', 'modifytimestamp_format')
+                modifytimestamp_format = conf.get_raw('ldap', 'modifytimestamp_format').replace('%%', '%')
                 if modifytimestamp_format == None:
                     modifytimestamp_format = "%Y%m%d%H%M%SZ"
 
@@ -1306,7 +1310,7 @@ class LDAP(pykolab.base.Base):
 
         _filter = "(&%s(modifytimestamp>=%s))" % (_filter,modified_after)
 
-        log.debug(_("Using filter %r") % (_filter), level=8)
+        log.debug(_("Synchronization is using filter %r") % (_filter), level=8)
 
         if not mode == 0:
             override_search = mode
@@ -2304,7 +2308,8 @@ class LDAP(pykolab.base.Base):
         # The list of naming contexts in the LDAP server
         attrs = self.get_entry_attributes("", ['namingContexts'])
 
-        naming_contexts = attrs['namingcontexts']
+        # Lower case of naming contexts - primarily for AD
+        naming_contexts = utils.normalize(attrs['namingcontexts'])
 
         if isinstance(naming_contexts, basestring):
             naming_contexts = [ naming_contexts ]
@@ -2643,6 +2648,9 @@ class LDAP(pykolab.base.Base):
 
         # Typical for Persistent Change Control EntryChangeNotification
         if kw.has_key('change_type'):
+
+            log.debug(_("change_type defined, typical for Persistent Change Control EntryChangeNotification"), level=5)
+
             change_type = None
 
             change_dict = {
@@ -2693,6 +2701,9 @@ class LDAP(pykolab.base.Base):
 
         # Typical for Paged Results Control
         elif kw.has_key('entry') and isinstance(kw['entry'], list):
+
+            log.debug(_("No change_type, typical for Paged Results Control"), level=5)
+
             for entry_dn,entry_attrs in kw['entry']:
                 # This is a referral
                 if entry_dn == None:
@@ -2703,7 +2714,7 @@ class LDAP(pykolab.base.Base):
                 for attr in entry_attrs.keys():
                     entry[attr.lower()] = entry_attrs[attr]
 
-                unique_attr = self.config_get('unique_attribute')
+                unique_attr = self.config_get('unique_attribute').lower()
                 entry['id'] = entry[unique_attr]
 
                 try:
@@ -2711,7 +2722,7 @@ class LDAP(pykolab.base.Base):
                 except:
                     entry['type'] = "unknown"
 
-                log.debug(_("Entry type: %s") % (entry['type']), level=8)
+                log.debug(_("Entry type for dn: %s is: %s") % (entry['dn'], entry['type']), level=8)
 
                 eval("self._change_none_%s(entry, None)" % (entry['type']))
 
@@ -2878,6 +2889,9 @@ class LDAP(pykolab.base.Base):
 
                 break
 
+            # Remove referrals
+            _result_data = [_e for _e in _result_data if _e[0] is not None]
+
             if callback:
                 callback(entry=_result_data)
 
diff --git a/pykolab/auth/ldap/cache.py b/pykolab/auth/ldap/cache.py
index 02c3831..b72e5ab 100644
--- a/pykolab/auth/ldap/cache.py
+++ b/pykolab/auth/ldap/cache.py
@@ -31,6 +31,8 @@ from sqlalchemy import desc
 from sqlalchemy import create_engine
 from sqlalchemy.orm import mapper
 
+from uuid import UUID
+
 try:
     from sqlalchemy.orm import relationship
 except:
@@ -63,7 +65,7 @@ class Entry(object):
         self.uniqueid = uniqueid
         self.result_attribute = result_attr
 
-        modifytimestamp_format = conf.get_raw('ldap', 'modifytimestamp_format')
+        modifytimestamp_format = conf.get_raw('ldap', 'modifytimestamp_format').replace('%%', '%')
         if modifytimestamp_format == None:
             modifytimestamp_format = "%Y%m%d%H%M%SZ"
 
@@ -110,52 +112,59 @@ def get_entry(domain, entry, update=True):
     _entry = None
 
     db = init_db(domain)
+
+    try:
+        _uniqueid = str(UUID(bytes_le=entry['id']))
+        log.debug(_("Entry uniqueid was converted from binary form to string: %s") % _uniqueid, level=8)
+    except ValueError:
+        _uniqueid = entry['id']
+
     try:
-        _entry = db.query(Entry).filter_by(uniqueid=entry['id']).first()
+        _entry = db.query(Entry).filter_by(uniqueid=_uniqueid).first()
     except sqlalchemy.exc.OperationalError, errmsg:
         db = init_db(domain,reinit=True)
     except sqlalchemy.exc.InvalidRequestError, errmsg:
         db = init_db(domain,reinit=True)
     finally:
-        _entry = db.query(Entry).filter_by(uniqueid=entry['id']).first()
+        _entry = db.query(Entry).filter_by(uniqueid=_uniqueid).first()
 
     if not update:
         return _entry
 
     if _entry == None:
-        log.debug(_("Inserting cache entry %r") % (entry['id']), level=8)
+        log.debug(_("Inserting cache entry %r") % (_uniqueid), level=8)
 
         if not entry.has_key(result_attribute):
             entry[result_attribute] = ''
 
         db.add(
                 Entry(
-                        entry['id'],
+                        _uniqueid,
                         entry[result_attribute],
                         entry['modifytimestamp']
                     )
             )
 
         db.commit()
-        _entry = db.query(Entry).filter_by(uniqueid=entry['id']).first()
+        _entry = db.query(Entry).filter_by(uniqueid=_uniqueid).first()
     else:
-        modifytimestamp_format = conf.get_raw('ldap', 'modifytimestamp_format')
+        modifytimestamp_format = conf.get_raw('ldap', 'modifytimestamp_format').replace('%%', '%')
         if modifytimestamp_format == None:
             modifytimestamp_format = "%Y%m%d%H%M%SZ"
 
         if not _entry.last_change.strftime(modifytimestamp_format) == entry['modifytimestamp']:
-            log.debug(_("Updating timestamp for cache entry %r") % (entry['id']), level=8)
+            log.debug(_("Updating timestamp for cache entry %r") % (_uniqueid), level=8)
             last_change = datetime.datetime.strptime(entry['modifytimestamp'], modifytimestamp_format)
             _entry.last_change = last_change
             db.commit()
-            _entry = db.query(Entry).filter_by(uniqueid=entry['id']).first()
+            _entry = db.query(Entry).filter_by(uniqueid=_uniqueid).first()
 
         if entry.has_key(result_attribute):
             if not _entry.result_attribute == entry[result_attribute]:
-                log.debug(_("Updating result_attribute for cache entry %r") % (entry['id']), level=8)
+                log.debug(_("Updating result_attribute for cache entry %r") % (_uniqueid), level=8)
                 _entry.result_attribute = entry[result_attribute]
                 db.commit()
-                _entry = db.query(Entry).filter_by(uniqueid=entry['id']).first()
+                _entry = db.query(Entry).filter_by(uniqueid=_uniqueid).first()
 
     return _entry
 
@@ -189,7 +198,7 @@ def init_db(domain,reinit=False):
     return db[domain]
 
 def last_modify_timestamp(domain):
-    modifytimestamp_format = conf.get_raw('ldap', 'modifytimestamp_format')
+    modifytimestamp_format = conf.get_raw('ldap', 'modifytimestamp_format').replace('%%', '%')
     if modifytimestamp_format == None:
         modifytimestamp_format = "%Y%m%d%H%M%SZ"
 
diff --git a/pykolab/constants.py.in b/pykolab/constants.py.in
index 779e598..9b06666 100644
--- a/pykolab/constants.py.in
+++ b/pykolab/constants.py.in
@@ -118,3 +118,9 @@ SUPPORTED_LDAP_CONTROLS = {
 #                'func': '_sync_repl'
 #            }
 #    }
+
+# Binay attributes which should not be stripped
+BINARY_ATTRS = (
+        'objectguid',
+        'objectsid'
+    )
diff --git a/pykolab/utils.py b/pykolab/utils.py
index 8bc82e5..9794a57 100644
--- a/pykolab/utils.py
+++ b/pykolab/utils.py
@@ -364,7 +364,12 @@ def normalize(_object):
                 if _object[key] is None:
                     continue
 
-                val = map(_strip, _object[key])
+                # Dont run strip anything from attributes which
+                # hold byte strings
+                if key.lower() in constants.BINARY_ATTRS:
+                    val = _object[key]
+                else:
+                    val = map(_strip, _object[key])
 
                 if len(val) == 1:
                     result[key.lower()] = ''.join(val)