tools/metadata/generator.go

package main

import (
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"sort"
	"strings"
	"sync"

	"android/soong/testing/code_metadata_internal_proto"
	"android/soong/testing/code_metadata_proto"
	"android/soong/testing/test_spec_proto"
	"google.golang.org/protobuf/proto"
)

type keyToLocksMap struct {
	locks sync.Map
}

func (kl *keyToLocksMap) GetLockForKey(key string) *sync.Mutex {
	mutex, _ := kl.locks.LoadOrStore(key, &sync.Mutex{})
	return mutex.(*sync.Mutex)
}

// Define a struct to hold the combination of team ID and multi-ownership flag for validation
type sourceFileAttributes struct {
	TeamID         string
	MultiOwnership bool
	Path           string
}

func getSortedKeys(syncMap *sync.Map) []string {
	var allKeys []string
	syncMap.Range(
		func(key, _ interface{}) bool {
			allKeys = append(allKeys, key.(string))
			return true
		},
	)

	sort.Strings(allKeys)
	return allKeys
}

// writeProtoToFile marshals a protobuf message and writes it to a file
func writeProtoToFile(outputFile string, message proto.Message) {
	data, err := proto.Marshal(message)
	if err != nil {
		log.Fatal(err)
	}
	file, err := os.Create(outputFile)
	if err != nil {
		log.Fatal(err)
	}
	defer file.Close()

	_, err = file.Write(data)
	if err != nil {
		log.Fatal(err)
	}
}

func readFileToString(filePath string) string {
	file, err := os.Open(filePath)
	if err != nil {
		log.Fatal(err)
	}
	defer file.Close()

	data, err := io.ReadAll(file)
	if err != nil {
		log.Fatal(err)
	}
	return string(data)
}

func writeNewlineToOutputFile(outputFile string) {
	file, err := os.Create(outputFile)
	data := "\n"
	if err != nil {
		log.Fatal(err)
	}
	defer file.Close()

	_, err = file.Write([]byte(data))
	if err != nil {
		log.Fatal(err)
	}
}

func processTestSpecProtobuf(
		filePath string, ownershipMetadataMap *sync.Map, keyLocks *keyToLocksMap,
		errCh chan error, wg *sync.WaitGroup,
) {
	defer wg.Done()

	fileContent := strings.TrimRight(readFileToString(filePath), "\n")
	testData := test_spec_proto.TestSpec{}
	err := proto.Unmarshal([]byte(fileContent), &testData)
	if err != nil {
		errCh <- err
		return
	}

	ownershipMetadata := testData.GetOwnershipMetadataList()
	for _, metadata := range ownershipMetadata {
		key := metadata.GetTargetName()
		lock := keyLocks.GetLockForKey(key)
		lock.Lock()

		value, loaded := ownershipMetadataMap.LoadOrStore(
			key, []*test_spec_proto.TestSpec_OwnershipMetadata{metadata},
		)
		if loaded {
			existingMetadata := value.([]*test_spec_proto.TestSpec_OwnershipMetadata)
			isDuplicate := false
			for _, existing := range existingMetadata {
				if metadata.GetTrendyTeamId() != existing.GetTrendyTeamId() {
					errCh <- fmt.Errorf(
						"Conflicting trendy team IDs found for %s at:\n%s with teamId"+
								": %s,\n%s with teamId: %s",
						key,
						metadata.GetPath(), metadata.GetTrendyTeamId(), existing.GetPath(),
						existing.GetTrendyTeamId(),
					)

					lock.Unlock()
					return
				}
				if metadata.GetTrendyTeamId() == existing.GetTrendyTeamId() && metadata.GetPath() == existing.GetPath() {
					isDuplicate = true
					break
				}
			}
			if !isDuplicate {
				existingMetadata = append(existingMetadata, metadata)
				ownershipMetadataMap.Store(key, existingMetadata)
			}
		}

		lock.Unlock()
	}
}

// processCodeMetadataProtobuf processes CodeMetadata protobuf files
func processCodeMetadataProtobuf(
		filePath string, ownershipMetadataMap *sync.Map, sourceFileMetadataMap *sync.Map, keyLocks *keyToLocksMap,
		errCh chan error, wg *sync.WaitGroup,
) {
	defer wg.Done()

	fileContent := strings.TrimRight(readFileToString(filePath), "\n")
	internalCodeData := code_metadata_internal_proto.CodeMetadataInternal{}
	err := proto.Unmarshal([]byte(fileContent), &internalCodeData)
	if err != nil {
		errCh <- err
		return
	}

	// Process each TargetOwnership entry
	for _, internalMetadata := range internalCodeData.GetTargetOwnershipList() {
		key := internalMetadata.GetTargetName()
		lock := keyLocks.GetLockForKey(key)
		lock.Lock()

		for _, srcFile := range internalMetadata.GetSourceFiles() {
			srcFileKey := srcFile
			srcFileLock := keyLocks.GetLockForKey(srcFileKey)
			srcFileLock.Lock()
			attributes := sourceFileAttributes{
				TeamID:         internalMetadata.GetTrendyTeamId(),
				MultiOwnership: internalMetadata.GetMultiOwnership(),
				Path:           internalMetadata.GetPath(),
			}

			existingAttributes, exists := sourceFileMetadataMap.Load(srcFileKey)
			if exists {
				existing := existingAttributes.(sourceFileAttributes)
				if attributes.TeamID != existing.TeamID && (!attributes.MultiOwnership || !existing.MultiOwnership) {
					errCh <- fmt.Errorf(
						"Conflict found for source file %s covered at %s with team ID: %s. Existing team ID: %s and path: %s."+
								" If multi-ownership is required, multiOwnership should be set to true in all test_spec modules using this target. "+
								"Multiple-ownership in general is discouraged though as it make infrastructure around android relying on this information pick up a random value when it needs only one.",
						srcFile, internalMetadata.GetPath(), attributes.TeamID, existing.TeamID, existing.Path,
					)
					srcFileLock.Unlock()
					lock.Unlock()
					return
				}
			} else {
				// Store the metadata if no conflict
				sourceFileMetadataMap.Store(srcFileKey, attributes)
			}
			srcFileLock.Unlock()
		}

		value, loaded := ownershipMetadataMap.LoadOrStore(
			key, []*code_metadata_internal_proto.CodeMetadataInternal_TargetOwnership{internalMetadata},
		)
		if loaded {
			existingMetadata := value.([]*code_metadata_internal_proto.CodeMetadataInternal_TargetOwnership)
			isDuplicate := false
			for _, existing := range existingMetadata {
				if internalMetadata.GetTrendyTeamId() == existing.GetTrendyTeamId() && internalMetadata.GetPath() == existing.GetPath() {
					isDuplicate = true
					break
				}
			}
			if !isDuplicate {
				existingMetadata = append(existingMetadata, internalMetadata)
				ownershipMetadataMap.Store(key, existingMetadata)
			}
		}

		lock.Unlock()
	}
}

func main() {
	inputFile := flag.String("inputFile", "", "Input file path")
	outputFile := flag.String("outputFile", "", "Output file path")
	rule := flag.String(
		"rule", "", "Metadata rule (Hint: test_spec or code_metadata)",
	)
	flag.Parse()

	if *inputFile == "" || *outputFile == "" || *rule == "" {
		fmt.Println("Usage: metadata -rule <rule> -inputFile <input file path> -outputFile <output file path>")
		os.Exit(1)
	}

	inputFileData := strings.TrimRight(readFileToString(*inputFile), "\n")
	filePaths := strings.Split(inputFileData, " ")
	if len(filePaths) == 1 && filePaths[0] == "" {
		writeNewlineToOutputFile(*outputFile)
		return
	}
	ownershipMetadataMap := &sync.Map{}
	keyLocks := &keyToLocksMap{}
	errCh := make(chan error, len(filePaths))
	var wg sync.WaitGroup

	switch *rule {
	case "test_spec":
		for _, filePath := range filePaths {
			wg.Add(1)
			go processTestSpecProtobuf(
				filePath, ownershipMetadataMap, keyLocks, errCh, &wg,
			)
		}

		wg.Wait()
		close(errCh)

		for err := range errCh {
			log.Fatal(err)
		}

		allKeys := getSortedKeys(ownershipMetadataMap)
		var allMetadata []*test_spec_proto.TestSpec_OwnershipMetadata

		for _, key := range allKeys {
			value, _ := ownershipMetadataMap.Load(key)
			metadataList := value.([]*test_spec_proto.TestSpec_OwnershipMetadata)
			allMetadata = append(allMetadata, metadataList...)
		}

		testSpec := &test_spec_proto.TestSpec{
			OwnershipMetadataList: allMetadata,
		}
		writeProtoToFile(*outputFile, testSpec)
		break
	case "code_metadata":
		sourceFileMetadataMap := &sync.Map{}
		for _, filePath := range filePaths {
			wg.Add(1)
			go processCodeMetadataProtobuf(
				filePath, ownershipMetadataMap, sourceFileMetadataMap, keyLocks, errCh, &wg,
			)
		}

		wg.Wait()
		close(errCh)

		for err := range errCh {
			log.Fatal(err)
		}

		sortedKeys := getSortedKeys(ownershipMetadataMap)
		allMetadata := make([]*code_metadata_proto.CodeMetadata_TargetOwnership, 0)
		for _, key := range sortedKeys {
			value, _ := ownershipMetadataMap.Load(key)
			metadata := value.([]*code_metadata_internal_proto.CodeMetadataInternal_TargetOwnership)
			for _, m := range metadata {
				targetName := m.GetTargetName()
				path := m.GetPath()
				trendyTeamId := m.GetTrendyTeamId()

				allMetadata = append(allMetadata, &code_metadata_proto.CodeMetadata_TargetOwnership{
					TargetName:   &targetName,
					Path:         &path,
					TrendyTeamId: &trendyTeamId,
					SourceFiles:  m.GetSourceFiles(),
				})
			}
		}

		finalMetadata := &code_metadata_proto.CodeMetadata{
			TargetOwnershipList: allMetadata,
		}
		writeProtoToFile(*outputFile, finalMetadata)
		break
	default:
		log.Fatalf("No specific processing implemented for rule '%s'.\n", *rule)
	}
}