burlap/dj.py
def install_sql(self, site=None, database='default', apps=None, stop_on_error=0, fn=None):
        """
        Installs all custom SQL.
        """
        #from burlap.db import load_db_set

        stop_on_error = int(stop_on_error)

        site = site or ALL

        name = database

        r = self.local_renderer
        paths = glob.glob(r.format(r.env.install_sql_path_template))

        apps = [_ for _ in (apps or '').split(',') if _.strip()]
        if self.verbose:
            print('install_sql.apps:', apps)

        def cmp_paths(d0, d1):
            if d0[1] and d0[1] in d1[2]:
                return -1
            if d1[1] and d1[1] in d0[2]:
                return +1
            return cmp(d0[0], d1[0])

        def get_paths(t):
            """
            Returns SQL file paths in an execution order that respect dependencies.
            """
            data = [] # [(path, view_name, content)]
            for path in paths:
                if fn and fn not in path:
                    continue
                parts = path.split('.')
                if len(parts) == 3 and parts[1] != t:
                    continue
                if not path.lower().endswith('.sql'):
                    continue
                content = open(path, 'r').read()
                matches = re.findall(r'[\s\t]+VIEW[\s\t]+([a-zA-Z0-9_]{3,})', content, flags=re.IGNORECASE)
                view_name = ''
                if matches:
                    view_name = matches[0]
                    print('Found view %s.' % view_name)
                data.append((path, view_name, content))
            for d in sorted(data, cmp=cmp_paths):
                yield d[0]

        def run_paths(paths, cmd_template, max_retries=3):
            r = self.local_renderer
            paths = list(paths)
            error_counts = defaultdict(int) # {path:count}
            terminal = set()
            if self.verbose:
                print('Checking %i paths.' % len(paths))
            while paths:
                path = paths.pop(0)
                if self.verbose:
                    print('path:', path)
                app_name = re.findall(r'/([^/]+)/sql/', path)[0]
                if apps and app_name not in apps:
                    self.vprint('skipping because app_name %s not in apps' % app_name)
                    continue
                with self.settings(warn_only=True):
                    if self.is_local:
                        r.env.sql_path = path
                    else:
                        r.env.sql_path = '/tmp/%s' % os.path.split(path)[-1]
                        r.put(local_path=path, remote_path=r.env.sql_path)
                    ret = r.run_or_local(cmd_template)
                    if ret and ret.return_code:

                        if stop_on_error:
                            raise Exception('Unable to execute file %s' % path)

                        error_counts[path] += 1
                        if error_counts[path] < max_retries:
                            paths.append(path)
                        else:
                            terminal.add(path)
            if terminal:
                print('%i files could not be loaded.' % len(terminal), file=sys.stderr)
                for path in sorted(list(terminal)):
                    print(path, file=sys.stderr)
                print(file=sys.stderr)

        if self.verbose:
            print('install_sql.db_engine:', r.env.db_engine)

        for _site, site_data in self.iter_sites(site=site, no_secure=True):

            self.set_db(name=name, site=_site)

            if 'postgres' in r.env.db_engine or 'postgis' in r.env.db_engine:
                paths = list(get_paths('postgresql'))
                run_paths(
                    paths=paths,
                    cmd_template="psql --host={db_host} --user={db_user} --no-password -d {db_name} -f {sql_path}")

            elif 'mysql' in r.env.db_engine:
                paths = list(get_paths('mysql'))
                run_paths(
                    paths=paths,
                    cmd_template="mysql -v -h {db_host} -u {db_user} -p'{db_password}' {db_name} < {sql_path}")

            else:
                raise NotImplementedError
def install_sql(self, site=None, database='default', apps=None, stop_on_error=0, fn=None):
        """
        Installs all custom SQL.
        """
        #from burlap.db import load_db_set

        stop_on_error = int(stop_on_error)

        site = site or ALL

        name = database

        r = self.local_renderer
        paths = glob.glob(r.format(r.env.install_sql_path_template))

        apps = [_ for _ in (apps or '').split(',') if _.strip()]
        if self.verbose:
            print('install_sql.apps:', apps)

        def cmp_paths(d0, d1):
            if d0[1] and d0[1] in d1[2]:
                return -1
            if d1[1] and d1[1] in d0[2]:
                return +1
            return cmp(d0[0], d1[0])

        def get_paths(t):
            """
            Returns SQL file paths in an execution order that respect dependencies.
            """
            data = [] # [(path, view_name, content)]
            for path in paths:
                if fn and fn not in path:
                    continue
                parts = path.split('.')
                if len(parts) == 3 and parts[1] != t:
                    continue
                if not path.lower().endswith('.sql'):
                    continue
                content = open(path, 'r').read()
                matches = re.findall(r'[\s\t]+VIEW[\s\t]+([a-zA-Z0-9_]{3,})', content, flags=re.IGNORECASE)
                view_name = ''
                if matches:
                    view_name = matches[0]
                    print('Found view %s.' % view_name)
                data.append((path, view_name, content))
            for d in sorted(data, cmp=cmp_paths):
                yield d[0]

        def run_paths(paths, cmd_template, max_retries=3):
            r = self.local_renderer
            paths = list(paths)
            error_counts = defaultdict(int) # {path:count}
            terminal = set()
            if self.verbose:
                print('Checking %i paths.' % len(paths))
            while paths:
                path = paths.pop(0)
                if self.verbose:
                    print('path:', path)
                app_name = re.findall(r'/([^/]+)/sql/', path)[0]
                if apps and app_name not in apps:
                    self.vprint('skipping because app_name %s not in apps' % app_name)
                    continue
                with self.settings(warn_only=True):
                    if self.is_local:
                        r.env.sql_path = path
                    else:
                        r.env.sql_path = '/tmp/%s' % os.path.split(path)[-1]
                        r.put(local_path=path, remote_path=r.env.sql_path)
                    ret = r.run_or_local(cmd_template)
                    if ret and ret.return_code:

                        if stop_on_error:
                            raise Exception('Unable to execute file %s' % path)

                        error_counts[path] += 1
                        if error_counts[path] < max_retries:
                            paths.append(path)
                        else:
                            terminal.add(path)
            if terminal:
                print('%i files could not be loaded.' % len(terminal), file=sys.stderr)
                for path in sorted(list(terminal)):
                    print(path, file=sys.stderr)
                print(file=sys.stderr)

        if self.verbose:
            print('install_sql.db_engine:', r.env.db_engine)

        for _site, site_data in self.iter_sites(site=site, no_secure=True):

            self.set_db(name=name, site=_site)

            if 'postgres' in r.env.db_engine or 'postgis' in r.env.db_engine:
                paths = list(get_paths('postgresql'))
                run_paths(
                    paths=paths,
                    cmd_template="psql --host={db_host} --user={db_user} --no-password -d {db_name} -f {sql_path}")

            elif 'mysql' in r.env.db_engine:
                paths = list(get_paths('mysql'))
                run_paths(
                    paths=paths,
                    cmd_template="mysql -v -h {db_host} -u {db_user} -p'{db_password}' {db_name} < {sql_path}")

            else:
                raise NotImplementedError
def install_sql ( self , site = None , database = 'default' , apps = None , stop_on_error = 0 , fn = None ) : #from burlap.db import load_db_set stop_on_error = int ( stop_on_error ) site = site or ALL name = database r = self . local_renderer paths = glob . glob ( r . format ( r . env . install_sql_path_template ) ) apps = [ _ for _ in ( apps or '' ) . split ( ',' ) if _ . strip ( ) ] if self . verbose : print ( 'install_sql.apps:' , apps ) def cmp_paths ( d0 , d1 ) : if d0 [ 1 ] and d0 [ 1 ] in d1 [ 2 ] : return - 1 if d1 [ 1 ] and d1 [ 1 ] in d0 [ 2 ] : return + 1 return cmp ( d0 [ 0 ] , d1 [ 0 ] ) def get_paths ( t ) : """
            Returns SQL file paths in an execution order that respect dependencies.
            """ data = [ ] # [(path, view_name, content)] for path in paths : if fn and fn not in path : continue parts = path . split ( '.' ) if len ( parts ) == 3 and parts [ 1 ] != t : continue if not path . lower ( ) . endswith ( '.sql' ) : continue content = open ( path , 'r' ) . read ( ) matches = re . findall ( r'[\s\t]+VIEW[\s\t]+([a-zA-Z0-9_]{3,})' , content , flags = re . IGNORECASE ) view_name = '' if matches : view_name = matches [ 0 ] print ( 'Found view %s.' % view_name ) data . append ( ( path , view_name , content ) ) for d in sorted ( data , cmp = cmp_paths ) : yield d [ 0 ] def run_paths ( paths , cmd_template , max_retries = 3 ) : r = self . local_renderer paths = list ( paths ) error_counts = defaultdict ( int ) # {path:count} terminal = set ( ) if self . verbose : print ( 'Checking %i paths.' % len ( paths ) ) while paths : path = paths . pop ( 0 ) if self . verbose : print ( 'path:' , path ) app_name = re . findall ( r'/([^/]+)/sql/' , path ) [ 0 ] if apps and app_name not in apps : self . vprint ( 'skipping because app_name %s not in apps' % app_name ) continue with self . settings ( warn_only = True ) : if self . is_local : r . env . sql_path = path else : r . env . sql_path = '/tmp/%s' % os . path . split ( path ) [ - 1 ] r . put ( local_path = path , remote_path = r . env . sql_path ) ret = r . run_or_local ( cmd_template ) if ret and ret . return_code : if stop_on_error : raise Exception ( 'Unable to execute file %s' % path ) error_counts [ path ] += 1 if error_counts [ path ] < max_retries : paths . append ( path ) else : terminal . add ( path ) if terminal : print ( '%i files could not be loaded.' % len ( terminal ) , file = sys . stderr ) for path in sorted ( list ( terminal ) ) : print ( path , file = sys . stderr ) print ( file = sys . stderr ) if self . verbose : print ( 'install_sql.db_engine:' , r . env . db_engine ) for _site , site_data in self . iter_sites ( site = site , no_secure = True ) : self . set_db ( name = name , site = _site ) if 'postgres' in r . env . db_engine or 'postgis' in r . env . db_engine : paths = list ( get_paths ( 'postgresql' ) ) run_paths ( paths = paths , cmd_template = "psql --host={db_host} --user={db_user} --no-password -d {db_name} -f {sql_path}" ) elif 'mysql' in r . env . db_engine : paths = list ( get_paths ( 'mysql' ) ) run_paths ( paths = paths , cmd_template = "mysql -v -h {db_host} -u {db_user} -p'{db_password}' {db_name} < {sql_path}" ) else : raise NotImplementedError
Installs all custom SQL.
Installs all custom SQL .