core/src/main/java/org/commonjava/indy/core/change/StoreContentListener.java
private void removeAllSupercededMemberContent( final ArtifactStore store,
                                                   final Map<ArtifactStore, ArtifactStore> changeMap )
    {
        StoreKey key = store.getKey();
        // we're only interested in groups, since only adjustments to group memberships can invalidate indexed content.
        if ( group == key.getType() )
        {
            List<StoreKey> newMembers = ( (Group) store ).getConstituents();
            logger.debug( "New members of: {} are: {}", store, newMembers );

            Group group = (Group) changeMap.get( store );
            List<StoreKey> oldMembers = group.getConstituents();
            logger.debug( "Old members of: {} are: {}", group, oldMembers );

            int commonSize = Math.min( newMembers.size(), oldMembers.size() );
            int divergencePoint;

            // look in the members that overlap in new/old groups and see if there are changes that would
            // indicate member reordering. If so, it might lead previously suppressed results to be prioritized,
            // which would invalidate part of the content index for the group.
            boolean foundDivergence = false;
            for ( divergencePoint = 0; divergencePoint < commonSize; divergencePoint++ )
            {
                logger.debug( "Checking for common member at index: {}", divergencePoint );
                if ( !oldMembers.get( divergencePoint ).equals( newMembers.get( divergencePoint ) ) )
                {
                    foundDivergence = true;
                    break;
                }
            }

            // [NOS-128]
            // 1. If membership has shrunk, we can remove origin-indexed paths, which will remove merged group content
            //      based on the removed member's content.
            // 2. If membership has grown, we should iterate new members' indexed content looking for mergable paths.
            //      For each of these, we need to removeIndexedStorePaths using the group and the mergable path.
            // [addendum]
            // 3. If membership is the same size but has been reordered, we need to iterate from the divergence point
            //    and invalidate the non-mergable files. This is because the reordering may change what artifacts
            //    should obscure which other physical artifacts.
            //
            // NOTE: In any case, once we isolate the changes, we need to handle matches in two ways:
            // 1. deleteTransfers()
            // 2. add the indexedStorePath to the removed Set so we can propagage their removal through any groups
            //      that include the one we're affecting directly here...using clearIndexedPathFrom() to do this.
            if ( !foundDivergence )
            {
                if ( newMembers.size() < oldMembers.size() )
                {
                    divergencePoint = commonSize;
                }
                else
                {
                    divergencePoint = newMembers.size();
                }
            }

            logger.debug( "group membership divergence point: {}", divergencePoint );

            Set<StoreKey> affectedMembers = new HashSet<>();
            boolean removeMergableOnly = divergencePoint >= oldMembers.size();

            // if we can iterate some old members that have been removed or reordered, invalidate the
            // group content index entries for those.
            if ( divergencePoint < oldMembers.size() )
            {
                for ( int i = divergencePoint; i < oldMembers.size(); i++ )
                {
                    affectedMembers.add( oldMembers.get( i ) );
                }
            }
            else
            {
                // for new added members, need to clear the indexed path with this group store for repo metadata merging
                // See [NOS-128]
                for ( int i = divergencePoint - 1; i >= commonSize; i-- )
                {
                    affectedMembers.add( newMembers.get( i ) );
                }
            }

            logger.debug( "Got members affected by membership divergence: {}", affectedMembers );
            if ( !affectedMembers.isEmpty() )
            {
                Set<Group> groups = new HashSet<>();
                groups.add( group );

                try
                {
                    groups.addAll( storeDataManager.query()
                                                   .packageType( group.getPackageType() )
                                                   .getGroupsAffectedBy( group.getKey() ) );
                }
                catch ( IndyDataException e )
                {
                    logger.error( String.format( "Cannot retrieve groups affected by: %s. Reason: %s", group.getKey(),
                                                 e.getMessage() ), e );
                }

                logger.debug( "Got affected groups: {}", groups );

                DrainingExecutorCompletionService<Integer> clearService =
                        new DrainingExecutorCompletionService<>( cleanupExecutor );

                final Predicate<? super String> mergableFilter = removeMergableOnly ? mergablePathStrings() : ( p ) -> true;

                // NOTE: We're NOT checking load for this executor, since this is an async process that is critical to
                // data integrity.
                affectedMembers.forEach( ( memberKey ) -> {
                    logger.debug( "Listing all {}paths in: {}", ( removeMergableOnly ? "mergeable " : "" ), memberKey );
                    listPathsAnd( memberKey, mergableFilter, p -> clearService.submit(
                            clearPathProcessor( p, memberKey, groups ) ) );

                } );

                drainAndCount( clearService, "store: " + store.getKey() );
            }
        }
private void removeAllSupercededMemberContent( final ArtifactStore store,
                                                   final Map<ArtifactStore, ArtifactStore> changeMap )
    {
        StoreKey key = store.getKey();
        // we're only interested in groups, since only adjustments to group memberships can invalidate indexed content.
        if ( group == key.getType() )
        {
            List<StoreKey> newMembers = ( (Group) store ).getConstituents();
            logger.debug( "New members of: {} are: {}", store, newMembers );

            Group group = (Group) changeMap.get( store );
            List<StoreKey> oldMembers = group.getConstituents();
            logger.debug( "Old members of: {} are: {}", group, oldMembers );

            int commonSize = Math.min( newMembers.size(), oldMembers.size() );
            int divergencePoint;

            // look in the members that overlap in new/old groups and see if there are changes that would
            // indicate member reordering. If so, it might lead previously suppressed results to be prioritized,
            // which would invalidate part of the content index for the group.
            boolean foundDivergence = false;
            for ( divergencePoint = 0; divergencePoint < commonSize; divergencePoint++ )
            {
                logger.debug( "Checking for common member at index: {}", divergencePoint );
                if ( !oldMembers.get( divergencePoint ).equals( newMembers.get( divergencePoint ) ) )
                {
                    foundDivergence = true;
                    break;
                }
            }

            // [NOS-128]
            // 1. If membership has shrunk, we can remove origin-indexed paths, which will remove merged group content
            //      based on the removed member's content.
            // 2. If membership has grown, we should iterate new members' indexed content looking for mergable paths.
            //      For each of these, we need to removeIndexedStorePaths using the group and the mergable path.
            // [addendum]
            // 3. If membership is the same size but has been reordered, we need to iterate from the divergence point
            //    and invalidate the non-mergable files. This is because the reordering may change what artifacts
            //    should obscure which other physical artifacts.
            //
            // NOTE: In any case, once we isolate the changes, we need to handle matches in two ways:
            // 1. deleteTransfers()
            // 2. add the indexedStorePath to the removed Set so we can propagage their removal through any groups
            //      that include the one we're affecting directly here...using clearIndexedPathFrom() to do this.
            if ( !foundDivergence )
            {
                if ( newMembers.size() < oldMembers.size() )
                {
                    divergencePoint = commonSize;
                }
                else
                {
                    divergencePoint = newMembers.size();
                }
            }

            logger.debug( "group membership divergence point: {}", divergencePoint );

            Set<StoreKey> affectedMembers = new HashSet<>();
            boolean removeMergableOnly = divergencePoint >= oldMembers.size();

            // if we can iterate some old members that have been removed or reordered, invalidate the
            // group content index entries for those.
            if ( divergencePoint < oldMembers.size() )
            {
                for ( int i = divergencePoint; i < oldMembers.size(); i++ )
                {
                    affectedMembers.add( oldMembers.get( i ) );
                }
            }
            else
            {
                // for new added members, need to clear the indexed path with this group store for repo metadata merging
                // See [NOS-128]
                for ( int i = divergencePoint - 1; i >= commonSize; i-- )
                {
                    affectedMembers.add( newMembers.get( i ) );
                }
            }

            logger.debug( "Got members affected by membership divergence: {}", affectedMembers );
            if ( !affectedMembers.isEmpty() )
            {
                Set<Group> groups = new HashSet<>();
                groups.add( group );

                try
                {
                    groups.addAll( storeDataManager.query()
                                                   .packageType( group.getPackageType() )
                                                   .getGroupsAffectedBy( group.getKey() ) );
                }
                catch ( IndyDataException e )
                {
                    logger.error( String.format( "Cannot retrieve groups affected by: %s. Reason: %s", group.getKey(),
                                                 e.getMessage() ), e );
                }

                logger.debug( "Got affected groups: {}", groups );

                DrainingExecutorCompletionService<Integer> clearService =
                        new DrainingExecutorCompletionService<>( cleanupExecutor );

                final Predicate<? super String> mergableFilter = removeMergableOnly ? mergablePathStrings() : ( p ) -> true;

                // NOTE: We're NOT checking load for this executor, since this is an async process that is critical to
                // data integrity.
                affectedMembers.forEach( ( memberKey ) -> {
                    logger.debug( "Listing all {}paths in: {}", ( removeMergableOnly ? "mergeable " : "" ), memberKey );
                    listPathsAnd( memberKey, mergableFilter, p -> clearService.submit(
                            clearPathProcessor( p, memberKey, groups ) ) );

                } );

                drainAndCount( clearService, "store: " + store.getKey() );
            }
        }
private void removeAllSupercededMemberContent ( final ArtifactStore store , final Map < ArtifactStore , ArtifactStore > changeMap ) { StoreKey key = store . getKey ( ) ; // we're only interested in groups, since only adjustments to group memberships can invalidate indexed content. if ( group == key . getType ( ) ) { List < StoreKey > newMembers = ( ( Group ) store ) . getConstituents ( ) ; logger . debug ( "New members of: {} are: {}" , store , newMembers ) ; Group group = ( Group ) changeMap . get ( store ) ; List < StoreKey > oldMembers = group . getConstituents ( ) ; logger . debug ( "Old members of: {} are: {}" , group , oldMembers ) ; int commonSize = Math . min ( newMembers . size ( ) , oldMembers . size ( ) ) ; int divergencePoint ; // look in the members that overlap in new/old groups and see if there are changes that would // indicate member reordering. If so, it might lead previously suppressed results to be prioritized, // which would invalidate part of the content index for the group. boolean foundDivergence = false ; for ( divergencePoint = 0 ; divergencePoint < commonSize ; divergencePoint ++ ) { logger . debug ( "Checking for common member at index: {}" , divergencePoint ) ; if ( ! oldMembers . get ( divergencePoint ) . equals ( newMembers . get ( divergencePoint ) ) ) { foundDivergence = true ; break ; } } // [NOS-128] // 1. If membership has shrunk, we can remove origin-indexed paths, which will remove merged group content //      based on the removed member's content. // 2. If membership has grown, we should iterate new members' indexed content looking for mergable paths. //      For each of these, we need to removeIndexedStorePaths using the group and the mergable path. // [addendum] // 3. If membership is the same size but has been reordered, we need to iterate from the divergence point //    and invalidate the non-mergable files. This is because the reordering may change what artifacts //    should obscure which other physical artifacts. // // NOTE: In any case, once we isolate the changes, we need to handle matches in two ways: // 1. deleteTransfers() // 2. add the indexedStorePath to the removed Set so we can propagage their removal through any groups //      that include the one we're affecting directly here...using clearIndexedPathFrom() to do this. if ( ! foundDivergence ) { if ( newMembers . size ( ) < oldMembers . size ( ) ) { divergencePoint = commonSize ; } else { divergencePoint = newMembers . size ( ) ; } } logger . debug ( "group membership divergence point: {}" , divergencePoint ) ; Set < StoreKey > affectedMembers = new HashSet <> ( ) ; boolean removeMergableOnly = divergencePoint >= oldMembers . size ( ) ; // if we can iterate some old members that have been removed or reordered, invalidate the // group content index entries for those. if ( divergencePoint < oldMembers . size ( ) ) { for ( int i = divergencePoint ; i < oldMembers . size ( ) ; i ++ ) { affectedMembers . add ( oldMembers . get ( i ) ) ; } } else { // for new added members, need to clear the indexed path with this group store for repo metadata merging // See [NOS-128] for ( int i = divergencePoint - 1 ; i >= commonSize ; i -- ) { affectedMembers . add ( newMembers . get ( i ) ) ; } } logger . debug ( "Got members affected by membership divergence: {}" , affectedMembers ) ; if ( ! affectedMembers . isEmpty ( ) ) { Set < Group > groups = new HashSet <> ( ) ; groups . add ( group ) ; try { groups . addAll ( storeDataManager . query ( ) . packageType ( group . getPackageType ( ) ) . getGroupsAffectedBy ( group . getKey ( ) ) ) ; } catch ( IndyDataException e ) { logger . error ( String . format ( "Cannot retrieve groups affected by: %s. Reason: %s" , group . getKey ( ) , e . getMessage ( ) ) , e ) ; } logger . debug ( "Got affected groups: {}" , groups ) ; DrainingExecutorCompletionService < Integer > clearService = new DrainingExecutorCompletionService <> ( cleanupExecutor ) ; final Predicate < ? super String > mergableFilter = removeMergableOnly ? mergablePathStrings ( ) : ( p ) - > true ; // NOTE: We're NOT checking load for this executor, since this is an async process that is critical to // data integrity. affectedMembers . forEach ( ( memberKey ) - > { logger . debug ( "Listing all {}paths in: {}" , ( removeMergableOnly ? "mergeable " : "" ) , memberKey ) ; listPathsAnd ( memberKey , mergableFilter , p -> clearService . submit ( clearPathProcessor ( p , memberKey , groups ) ) ) ; } ) ; drainAndCount ( clearService , "store: " + store . getKey ( ) ) ; } }
TODO: If we find points where a new HostedRepository is added, we should be using its comprehensive index to minimize the index damage to the group.
TODO : If we find points where a new HostedRepository is added we should be using its comprehensive index to minimize the index damage to the group .