tsp, internal model generate to implementation package

extension-base/src/main/java/com/azure/autorest/extension/base/model/codemodel/SchemaContext.java

@@ -14,11 +14,13 @@ public enum SchemaContext {
 
     EXCEPTION("exception"),
 
-    CONVENIENCE_API("convenience-api"),
+    PUBLIC("public"),
 
     PAGED("paged"),
 
-    ANONYMOUS("anonymous");
+    ANONYMOUS("anonymous"),
+
+    INTERNAL("internal");
 
     private final String value;
     private final static Map<String, SchemaContext> CONSTANTS = new HashMap<>();

javagen/src/main/java/com/azure/autorest/mapper/MapperUtils.java

@@ -9,6 +9,7 @@
 import com.azure.autorest.extension.base.model.codemodel.Operation;
 import com.azure.autorest.extension.base.model.codemodel.Response;
 import com.azure.autorest.extension.base.model.codemodel.Schema;
+import com.azure.autorest.extension.base.model.codemodel.SchemaContext;
 import com.azure.autorest.extension.base.plugin.JavaSettings;
 import com.azure.autorest.model.clientmodel.ClassType;
 import com.azure.autorest.model.clientmodel.ClientEnumValue;
@@ -35,11 +36,13 @@ static IType createEnumType(ChoiceSchema enumType, boolean expandable) {
         if (enumTypeName == null || enumTypeName.isEmpty() || enumTypeName.equals("enum")) {
             return ClassType.String;
         } else {
-            String enumSubpackage = settings.getModelsSubpackage();
+            String enumPackage = settings.getPackage(settings.getModelsSubpackage());
             if (settings.isCustomType(enumTypeName)) {
-                enumSubpackage = settings.getCustomTypesSubpackage();
+                enumPackage = settings.getPackage(settings.getCustomTypesSubpackage());
+            } else if (settings.isDataPlaneClient() && (enumType.getUsage() != null && enumType.getUsage().contains(SchemaContext.INTERNAL))) {
+                // internal type, which is not exposed to user
+                enumPackage = settings.getPackage(settings.getImplementationSubpackage(), settings.getModelsSubpackage());
             }
-            String enumPackage = settings.getPackage(enumSubpackage);
 
             String summary = enumType.getSummary();
             String description = enumType.getLanguage().getJava() == null ? null : enumType.getLanguage().getJava().getDescription();

javagen/src/main/java/com/azure/autorest/mapper/ObjectMapper.java

@@ -55,9 +55,11 @@ private ClassType createClassType(ObjectSchema compositeType) {
         } else if (settings.isFluent() && compositeType.isFlattenedSchema()) {
             // put class of flattened type to implementation package
             classPackage = settings.getPackage(settings.getFluentModelsSubpackage());
-        } else if (settings.isDataPlaneClient() && isPageModel(compositeType)) {
+        } else if (settings.isDataPlaneClient() && (isPageModel(compositeType) || isInternalModel(compositeType))) {
             // put class of Page<> type to implementation package
-            // For Cadl, these are not generated to class
+            // For TypeSpec, these are not generated to class
+
+            // same for internal type, which is not exposed to user
             classPackage = settings.getPackage(settings.getImplementationSubpackage(), settings.getModelsSubpackage());
         } else {
             classPackage = settings.getPackage(settings.getModelsSubpackage());
@@ -107,17 +109,16 @@ protected boolean isInnerModel(ObjectSchema compositeType) {
      * Extension for Page model.
      * <p>
      * Page model does not need to be exposed to user, as it is internal wire data that will be converted to PagedFlux or PagedIterable.
-     * Check in Cadl.
+     * Check in TypeSpec.
      *
      * @param compositeType object type
      * @return whether the type is a Page model.
      */
-    protected boolean isPageModel(ObjectSchema compositeType) {
-        boolean ret = false;
+    private static boolean isPageModel(ObjectSchema compositeType) {
+        return compositeType.getUsage() != null && compositeType.getUsage().contains(SchemaContext.PAGED);
+    }
 
-        if (compositeType.getUsage() != null && compositeType.getUsage().contains(SchemaContext.PAGED)) {
-            ret = true;
-        }
-        return ret;
+    private static boolean isInternalModel(ObjectSchema compositeType) {
+        return compositeType.getUsage() != null && compositeType.getUsage().contains(SchemaContext.INTERNAL);
     }
 }

javagen/src/main/java/com/azure/autorest/mapper/UnionMapper.java

@@ -4,6 +4,7 @@
 package com.azure.autorest.mapper;
 
 import com.azure.autorest.extension.base.model.codemodel.OrSchema;
+import com.azure.autorest.extension.base.model.codemodel.SchemaContext;
 import com.azure.autorest.extension.base.plugin.JavaSettings;
 import com.azure.autorest.model.clientmodel.ClassType;
 import com.azure.autorest.model.clientmodel.IType;
@@ -40,6 +41,11 @@ private ClassType createClassType(OrSchema compositeType) {
             ? settings.getPackage(settings.getCustomTypesSubpackage())
             : settings.getPackage(settings.getModelsSubpackage());
 
+        if (settings.isDataPlaneClient() && (compositeType.getUsage() != null && compositeType.getUsage().contains(SchemaContext.INTERNAL))) {
+            // internal type, which is not exposed to user
+            classPackage = settings.getPackage(settings.getImplementationSubpackage(), settings.getModelsSubpackage());
+        }
+
         return new ClassType.Builder()
             .packageName(classPackage)
             .name(className)

javagen/src/main/java/com/azure/autorest/model/clientmodel/ImplementationDetails.java

@@ -35,12 +35,12 @@ public enum Usage {
         EXCEPTION("exception"),
 
         /**
-         * Model used in input or output of methods marked as convenience API.
+         * Public model.
          * <p>
-         * In DPG, it means the model need to be written to Java class.
-         * Else, it may only exist in memory for Javadoc purpose.
+         * Usually it means that the model is used in input or output of methods marked as convenience API (and that API is not marked as internal).
+         * Codegen should generate the class in models package.
          */
-        CONVENIENCE_API("convenience-api"),
+        PUBLIC("public"),
 
         /**
          * Model used in paged response.
@@ -63,7 +63,14 @@ public enum Usage {
          * Javadoc or test/sample generation will still need to process the model.
          * Codegen likely need to have additional "require" clause in module-info.java, and additional dependency in pom.xml.
          */
-        EXTERNAL("external");
+        EXTERNAL("external"),
+
+        /**
+         * Internal model.
+         * <p>
+         * Codegen should generate the class in implementation package.
+         */
+        INTERNAL("internal");
 
         private final static Map<String, Usage> CONSTANTS = new HashMap<>();
         static {
@@ -115,10 +122,17 @@ public Set<Usage> getUsages() {
     }
 
     /**
-     * @return whether the model used for convenience method, or the method requires a convenience method.
+     * @return whether the model need to be generated for public use.
+     */
+    public boolean isPublic() {
+        return usages.contains(Usage.PUBLIC);
+    }
+
+    /**
+     * @return whether the model need to be generated for internal use.
      */
-    public boolean isConvenienceMethod() {
-        return usages.contains(Usage.CONVENIENCE_API);
+    public boolean isInternal() {
+        return usages.contains(Usage.INTERNAL);
     }
 
     public boolean isInput() {

typespec-extension/src/code-model-builder.ts

@@ -244,7 +244,7 @@ export class CodeModelBuilder {
 
     this.processModels(clients);
 
-    this.codeModel.schemas.objects?.forEach((it) => this.propagateSchemaUsage(it));
+    this.processSchemaUsage();
 
     if (this.options.namer) {
       this.codeModel = new PreNamer(this.codeModel).init().process();
@@ -264,7 +264,7 @@ export class CodeModelBuilder {
         } else {
           const schema = this.processSchema(it.type, it.name);
           this.trackSchemaUsage(schema, {
-            usage: [SchemaContext.Input, SchemaContext.Output /*SchemaContext.ConvenienceApi*/],
+            usage: [SchemaContext.Input, SchemaContext.Output /*SchemaContext.Public*/],
           });
           parameter = new Parameter(it.name, this.getDoc(it), schema, {
             implementation: ImplementationLocation.Client,
@@ -374,9 +374,10 @@ export class CodeModelBuilder {
   private processModels(clients: SdkClient[]) {
     const processedModels: Set<Type> = new Set();
     for (const client of clients) {
-      const models = client.service.models;
+      const models: (Model | Enum)[] = Array.from(client.service.models.values());
+      Array.from(client.service.enums.values()).forEach((it) => models.push(it));
 
-      for (const model of models.values()) {
+      for (const model of models) {
         if (!processedModels.has(model)) {
           const access = getAccess(model);
           if (access === "public") {
@@ -393,7 +394,13 @@ export class CodeModelBuilder {
             const schema = this.processSchema(model, model.name);
 
             this.trackSchemaUsage(schema, {
-              usage: [SchemaContext.ConvenienceApi],
+              usage: [SchemaContext.Public],
+            });
+          } else if (access === "internal") {
+            const schema = this.processSchema(model, model.name);
+
+            this.trackSchemaUsage(schema, {
+              usage: [SchemaContext.Internal],
             });
           }
 
@@ -412,6 +419,38 @@ export class CodeModelBuilder {
     }
   }
 
+  private processSchemaUsage() {
+    this.codeModel.schemas.objects?.forEach((it) => this.propagateSchemaUsage(it));
+
+    // post process for schema usage
+    this.codeModel.schemas.objects?.forEach((it) => this.resolveSchemaUsage(it));
+    this.codeModel.schemas.groups?.forEach((it) => this.resolveSchemaUsage(it));
+    this.codeModel.schemas.choices?.forEach((it) => this.resolveSchemaUsage(it));
+    this.codeModel.schemas.sealedChoices?.forEach((it) => this.resolveSchemaUsage(it));
+    this.codeModel.schemas.ors?.forEach((it) => this.resolveSchemaUsage(it));
+    this.codeModel.schemas.constants?.forEach((it) => this.resolveSchemaUsage(it));
+  }
+
+  private resolveSchemaUsage(schema: Schema) {
+    if (
+      schema instanceof ObjectSchema ||
+      schema instanceof GroupSchema ||
+      schema instanceof ChoiceSchema ||
+      schema instanceof SealedChoiceSchema ||
+      schema instanceof OrSchema ||
+      schema instanceof ConstantSchema
+    ) {
+      const schemaUsage: SchemaContext[] | undefined = schema.usage;
+      // Public override Internal
+      if (schemaUsage?.includes(SchemaContext.Public)) {
+        const index = schemaUsage.indexOf(SchemaContext.Internal);
+        if (index >= 0) {
+          schemaUsage.splice(index, 1);
+        }
+      }
+    }
+  }
+
   private processClients(): SdkClient[] {
     const clients = listClients(this.sdkContext);
     for (const client of clients) {
@@ -557,8 +596,12 @@ export class CodeModelBuilder {
       },
     });
 
+    // TODO (weidxu): temporary disable codeModelOperation.internalApi
+    // codeModelOperation.internalApi = this.isInternal(this.sdkContext, operation);
+    const internalApi = this.isInternal(this.sdkContext, operation);
+
     const convenienceApiName = this.getConvenienceApiName(operation);
-    let generateConvenienceApi: boolean = !!convenienceApiName && !this.isInternal(this.sdkContext, operation);
+    let generateConvenienceApi: boolean = !!convenienceApiName && !internalApi; // at present, internalApi means not convenienceApi. this could change.
 
     let apiComment: string | undefined = undefined;
     if (generateConvenienceApi) {
@@ -737,14 +780,18 @@ export class CodeModelBuilder {
       // track usage
       if (pollingSchema) {
         this.trackSchemaUsage(pollingSchema, { usage: [SchemaContext.Output] });
-        if (op.convenienceApi) {
-          this.trackSchemaUsage(pollingSchema, { usage: [SchemaContext.ConvenienceApi] });
+        if (op.internalApi) {
+          this.trackSchemaUsage(pollingSchema, { usage: [SchemaContext.Internal] });
+        } else if (op.convenienceApi) {
+          this.trackSchemaUsage(pollingSchema, { usage: [SchemaContext.Public] });
         }
       }
       if (finalSchema) {
         this.trackSchemaUsage(finalSchema, { usage: [SchemaContext.Output] });
-        if (op.convenienceApi) {
-          this.trackSchemaUsage(finalSchema, { usage: [SchemaContext.ConvenienceApi] });
+        if (op.internalApi) {
+          this.trackSchemaUsage(pollingSchema, { usage: [SchemaContext.Internal] });
+        } else if (op.convenienceApi) {
+          this.trackSchemaUsage(finalSchema, { usage: [SchemaContext.Public] });
         }
       }
 
@@ -902,8 +949,10 @@ export class CodeModelBuilder {
 
       this.trackSchemaUsage(schema, { usage: [SchemaContext.Input] });
 
-      if (op.convenienceApi) {
-        this.trackSchemaUsage(schema, { usage: [SchemaContext.ConvenienceApi] });
+      if (op.internalApi) {
+        this.trackSchemaUsage(schema, { usage: [SchemaContext.Internal] });
+      } else if (op.convenienceApi) {
+        this.trackSchemaUsage(schema, { usage: [SchemaContext.Public] });
       }
 
       if (param.name.toLowerCase() === "content-type") {
@@ -1053,7 +1102,12 @@ export class CodeModelBuilder {
           },
         );
 
-        this.trackSchemaUsage(requestConditionsSchema, { usage: [SchemaContext.Input, SchemaContext.ConvenienceApi] });
+        this.trackSchemaUsage(requestConditionsSchema, { usage: [SchemaContext.Input] });
+        if (op.internalApi) {
+          this.trackSchemaUsage(requestConditionsSchema, { usage: [SchemaContext.Internal] });
+        } else if (op.convenienceApi) {
+          this.trackSchemaUsage(requestConditionsSchema, { usage: [SchemaContext.Public] });
+        }
 
         // update group schema for properties
         for (const parameter of request.parameters) {
@@ -1112,8 +1166,10 @@ export class CodeModelBuilder {
 
     this.trackSchemaUsage(schema, { usage: [SchemaContext.Input] });
 
-    if (op.convenienceApi) {
-      this.trackSchemaUsage(schema, { usage: [SchemaContext.ConvenienceApi] });
+    if (op.internalApi) {
+      this.trackSchemaUsage(schema, { usage: [SchemaContext.Internal] });
+    } else if (op.convenienceApi) {
+      this.trackSchemaUsage(schema, { usage: [SchemaContext.Public] });
     }
 
     if (!schema.language.default.name && schema instanceof ObjectSchema) {
@@ -1203,7 +1259,7 @@ export class CodeModelBuilder {
             );
           });
 
-          this.trackSchemaUsage(optionBagSchema, { usage: [SchemaContext.Input, SchemaContext.ConvenienceApi] });
+          this.trackSchemaUsage(optionBagSchema, { usage: [SchemaContext.Input, SchemaContext.Public] });
 
           // option bag parameter
           const optionBagParameter = new Parameter(
@@ -1375,8 +1431,10 @@ export class CodeModelBuilder {
       if (response instanceof SchemaResponse) {
         this.trackSchemaUsage(response.schema, { usage: [SchemaContext.Output] });
 
-        if (trackConvenienceApi) {
-          this.trackSchemaUsage(response.schema, { usage: [SchemaContext.ConvenienceApi] });
+        if (op.internalApi) {
+          this.trackSchemaUsage(response.schema, { usage: [SchemaContext.Internal] });
+        } else if (trackConvenienceApi) {
+          this.trackSchemaUsage(response.schema, { usage: [SchemaContext.Public] });
         }
       }
     }
@@ -2318,7 +2376,7 @@ export class CodeModelBuilder {
       }
 
       processedSchemas.add(schema);
-      if (schema instanceof ObjectSchema) {
+      if (schema instanceof ObjectSchema || schema instanceof GroupSchema) {
         if (schemaUsage.usage || schemaUsage.serializationFormats) {
           schema.properties?.forEach((p) => {
             if (p.readOnly && schemaUsage.usage?.includes(SchemaContext.Input)) {
@@ -2332,11 +2390,13 @@ export class CodeModelBuilder {
             }
           });
 
-          schema.parents?.all?.forEach((p) => innerApplySchemaUsage(p, schemaUsage));
-          schema.parents?.immediate?.forEach((p) => innerApplySchemaUsage(p, schemaUsage));
+          if (schema instanceof ObjectSchema) {
+            schema.parents?.all?.forEach((p) => innerApplySchemaUsage(p, schemaUsage));
+            schema.parents?.immediate?.forEach((p) => innerApplySchemaUsage(p, schemaUsage));
 
-          schema.children?.all?.forEach((c) => innerApplySchemaUsage(c, schemaUsage));
-          schema.children?.immediate?.forEach((c) => innerApplySchemaUsage(c, schemaUsage));
+            schema.children?.all?.forEach((c) => innerApplySchemaUsage(c, schemaUsage));
+            schema.children?.immediate?.forEach((c) => innerApplySchemaUsage(c, schemaUsage));
+          }
 
           // Object.values(schema.discriminator?.all ?? {}).forEach((d) => {
           //   innerApplySchemaUsage(d, schemaUsage);

typespec-extension/src/common/operation.ts

@@ -51,9 +51,12 @@ export interface Operation extends Aspect {
   /** the name of convenience API */
   convenienceApi?: ConvenienceApi;
 
-  /** a boolean to represent should generate protocol api or not */
+  /** generate protocol api or not */
   generateProtocolApi?: boolean;
 
+  /** genrate as internal API */
+  internalApi?: boolean;
+
   /** the long-running operation metadata */
   lroMetadata?: LongRunningMetadata;
 }

typespec-extension/src/common/schemas/usage.ts

@@ -10,8 +10,11 @@ export enum SchemaContext {
   /** Schema is used as an exception from an operation. */
   Exception = "exception",
 
-  /** Schema is used from the operation for generating convenience API. */
-  ConvenienceApi = "convenience-api",
+  /** Schema is used from the operation for generating convenience API. And it is public */
+  Public = "public",
+
+  /** Schema is used for internal API, or dev purpose. Not exposed to user. */
+  Internal = "internal",
 
   /** Schema is used from the pageable operation. This usage does not propagate. */
   Paged = "paged",