Rebased cnb buildpack updates

app/actions/app_apply_manifest.rb

@@ -59,6 +59,7 @@ def apply(app_guid, message)
 
       app_update_message = message.app_update_message
       lifecycle = AppLifecycleProvider.provide_for_update(app_update_message, app)
+
       AppUpdate.new(@user_audit_info, manifest_triggered: true).update(app, app_update_message, lifecycle)
 
       update_routes(app, message)

app/controllers/v3/buildpacks_controller.rb

@@ -77,7 +77,7 @@ def upload
 
     unauthorized! unless permission_queryer.can_write_globally?
 
-    message = BuildpackUploadMessage.create_from_params(hashed_params[:body])
+    message = BuildpackUploadMessage.create_from_params(hashed_params[:body], buildpack.lifecycle)
     combine_messages(message.errors.full_messages) unless message.valid?
 
     unprocessable!('Buildpack is locked') if buildpack.locked

app/controllers/v3/space_manifests_controller.rb

@@ -16,6 +16,7 @@ def apply_manifest
     can_write_space(space)
 
     messages = parsed_app_manifests.map { |app_manifest| AppManifestMessage.create_from_yml(app_manifest) }
+
     errors = messages.each_with_index.flat_map { |message, i| errors_for_message(message, i) }
     compound_error!(errors) unless errors.empty?
 

app/fetchers/buildpack_list_fetcher.rb

@@ -19,7 +19,8 @@ def filter(message, dataset)
         dataset = dataset.where(name: message.names) if message.requested?(:names)
 
         dataset = NullFilterQueryGenerator.add_filter(dataset, :stack, message.stacks) if message.requested?(:stacks)
-        dataset = dataset.where(lifecycle: message.requested?(:lifecycle) ? message.lifecycle : Config.config.get(:default_app_lifecycle))
+
+        dataset = dataset.where(lifecycle: message.lifecycle) if message.requested?(:lifecycle)
 
         if message.requested?(:label_selector)
           dataset = LabelSelectorQueryGenerator.add_selector_queries(

app/messages/app_manifest_message.rb

@@ -123,16 +123,32 @@ def audit_hash
     end
 
     def app_lifecycle_hash
-      lifecycle_data = if requested?(:lifecycle) && @lifecycle == 'cnb'
-                         cnb_lifecycle_data
+      lifecycle_type = if requested?(:lifecycle) && @lifecycle == 'cnb'
+                         Lifecycles::CNB
+                       elsif requested?(:lifecycle) && @lifecycle == 'buildpack'
+                         Lifecycles::BUILDPACK
                        elsif requested?(:docker)
-                         docker_lifecycle_data
-                       elsif requested?(:buildpacks) || requested?(:buildpack) || requested?(:stack)
-                         buildpacks_lifecycle_data
+                         Lifecycles::DOCKER
                        end
 
+      data = {
+        buildpacks: requested_buildpacks,
+        stack: @stack,
+        credentials: @cnb_credentials
+      }.compact
+
+      if lifecycle_type == Lifecycles::DOCKER
+        lifecycle = {
+          type: Lifecycles::DOCKER
+        }
+      elsif lifecycle_type || data.present?
+        lifecycle = {}
+        lifecycle[:type] = lifecycle_type if lifecycle_type.present?
+        lifecycle[:data] = data if data.present?
+      end
+
       {
-        lifecycle: lifecycle_data,
+        lifecycle: lifecycle,
         metadata: requested?(:metadata) ? metadata : nil
       }.compact
     end
@@ -279,31 +295,6 @@ def service_bindings_attribute_mapping
       mapping
     end
 
-    def docker_lifecycle_data
-      { type: Lifecycles::DOCKER }
-    end
-
-    def cnb_lifecycle_data
-      {
-        type: Lifecycles::CNB,
-        data: {
-          buildpacks: requested_buildpacks,
-          stack: @stack,
-          credentials: @cnb_credentials
-        }.compact
-      }
-    end
-
-    def buildpacks_lifecycle_data
-      {
-        type: Lifecycles::BUILDPACK,
-        data: {
-          buildpacks: requested_buildpacks,
-          stack: @stack
-        }.compact
-      }
-    end
-
     def should_autodetect?(buildpack)
       buildpack == 'default' || buildpack == 'null' || buildpack.nil?
     end

app/messages/buildpack_upload_message.rb

@@ -3,6 +3,7 @@
 module VCAP::CloudController
   GZIP_MIME = Regexp.new("\x1F\x8B\x08".force_encoding('binary'))
   ZIP_MIME = Regexp.new("PK\x03\x04".force_encoding('binary'))
+  CNB_MIME = Regexp.new("\x75\x73\x74\x61\x72\x00\x30\x30".force_encoding('binary'))
 
   class BuildpackUploadMessage < BaseMessage
     class MissingFilePathError < StandardError; end
@@ -16,8 +17,15 @@ class MissingFilePathError < StandardError; end
     validate :is_not_empty
     validate :missing_file_path
 
-    def self.create_from_params(params)
-      BuildpackUploadMessage.new(params.dup.symbolize_keys)
+    attr_reader :lifecycle
+
+    def initialize(params, lifecycle)
+      @lifecycle = lifecycle
+      super(params)
+    end
+
+    def self.create_from_params(params, lifecycle)
+      BuildpackUploadMessage.new(params.dup.symbolize_keys, lifecycle)
     end
 
     def nginx_fields
@@ -51,9 +59,18 @@ def is_archive
 
       mime_bits = File.read(bits_path, 4)
 
-      return if mime_bits =~ /^#{VCAP::CloudController::GZIP_MIME}/ || mime_bits =~ /^#{VCAP::CloudController::ZIP_MIME}/
+      if lifecycle == VCAP::CloudController::Lifecycles::BUILDPACK
+        return if mime_bits =~ /^#{VCAP::CloudController::ZIP_MIME}/
+
+        errors.add(:base, "#{bits_name} is not a zip file. Buildpacks of lifecycle \"#{lifecycle}\" must be valid zip files.")
+      elsif lifecycle == VCAP::CloudController::Lifecycles::CNB
+        return if mime_bits =~ /^#{VCAP::CloudController::GZIP_MIME}/
+
+        mime_bits_at_offset = File.read(bits_path, 8, 257)
+        return if mime_bits_at_offset =~ /^#{VCAP::CloudController::CNB_MIME}/
 
-      errors.add(:base, "#{bits_name} is not a zip or gzip archive")
+        errors.add(:base, "#{bits_name} is not a gzip archive or cnb file. Buildpacks of lifecycle \"#{lifecycle}\" must be valid gzip archives or cnb files.")
+      end
     end
 
     def missing_file_path

app/messages/buildpacks_list_message.rb

@@ -21,7 +21,8 @@ class BuildpacksListMessage < MetadataListMessage
 
     def initialize(params={})
       super
-      pagination_options.default_order_by = :position
+      pagination_options.default_order_by = :lifecycle
+      pagination_options.secondary_default_order_by = :position
     end
 
     def self.from_params(params)
@@ -33,7 +34,7 @@ def to_param_hash
     end
 
     def valid_order_by_values
-      super << :position
+      super + %i[position lifecycle]
     end
   end
 

docs/v3/source/includes/api_resources/_buildpacks.erb

@@ -8,6 +8,7 @@
     "filename": null,
     "stack": "windows64",
     "position": 42,
+    "lifecycle": "buildpack",
     "enabled": true,
     "locked": false,
     "metadata": {
@@ -52,6 +53,7 @@
         "filename": null,
         "stack": "my-stack",
         "position": 1,
+        "lifecycle": "cnb",
         "enabled": true,
         "locked": false,
         "metadata": {

docs/v3/source/includes/resources/apps/_update.md.erb

@@ -39,7 +39,7 @@ Content-Type: application/json
 Name | Type | Description
 ---- | ---- | -----------
 **name** | _string_ | Name of the app
-**lifecycle** | [_lifecycle object_](#the-lifecycle-object) | Lifecycle to be used when updating the app; note: `data` is a required field in lifecycle if lifecycle is updated
+**lifecycle** | [_lifecycle object_](#the-lifecycle-object) | Lifecycle to be used when updating the app; note: `data` is a required field in lifecycle if lifecycle is updated. `type` may NOT be changed from its current value.
 **metadata.labels** | [_label object_](#labels) | Labels applied to the app
 **metadata.annotations**  | [_annotation object_](#annotations) | Annotations applied to the app
 

docs/v3/source/includes/resources/buildpacks/_create.md.erb

@@ -44,6 +44,7 @@ Name         | Type      | Description
 -----------  | --------  | -----------------------------------------------------------------------------  | -------
 **stack**    | _string_  | The name of the stack that the buildpack will use                             | null
 **position** | _integer_ | The order in which the buildpacks are checked during buildpack auto-detection | 1
+**lifecycle**| _string_  | The version of buildpack the buildpack will use. `buildpack` indicates [Classic Buildpacks](https://docs.cloudfoundry.org/buildpacks/classic.html). `cnb` indicates [Cloud Native Buildpacks](https://docs.cloudfoundry.org/buildpacks/cnb/) | buildpack
 **enabled**  | _boolean_ | Whether or not the buildpack will be used for staging                          | true
 **locked**   | _boolean_ | Whether or not the buildpack is locked to prevent updating the bits            | false
 **metadata.labels** | [_label object_](#labels) | Labels applied to the buildpack

docs/v3/source/includes/resources/buildpacks/_list.md.erb

@@ -34,7 +34,8 @@ Name | Type | Description
 **per_page** | _integer_ | Number of results per page; <br>valid values are 1 through 5000
 **names** | _list of strings_ | Comma-delimited list of buildpack names to filter by
 **stacks**| _list of strings_ | Comma-delimited list of stack names to filter by
-**order_by** | _string_ | Value to sort by; defaults to ascending. Prepend with `-` to sort descending. <br>Valid values are `created_at`, `updated_at`, and `position`
+**lifecycle**| _string_ | Type of buildpack. Valid values are `buildpack` and `cnb`
+**order_by** | _string_ | Value to sort by; defaults to ascending. Prepend with `-` to sort descending. <br>Valid values are `created_at`, `updated_at`, `lifecycle`, and `position`
 **label_selector** | _string_ | A query string containing a list of [label selector](#labels-and-selectors) requirements
 **created_ats** | _[timestamp](#timestamps)_ | Timestamp to filter by. When filtering on equality, several comma-delimited timestamps may be passed. Also supports filtering with [relational operators](#relational-operators)
 **updated_ats** | _[timestamp](#timestamps)_ | Timestamp to filter by. When filtering on equality, several comma-delimited timestamps may be passed. Also supports filtering with [relational operators](#relational-operators)

docs/v3/source/includes/resources/buildpacks/_object.md.erb

@@ -15,7 +15,8 @@ Name           | Type                     | Description
 **updated_at** | _[timestamp](#timestamps)_               | The time with zone when the object was last updated
 **name**       | _string_                 | The name of the buildpack; to be used by app buildpack field (only alphanumeric characters)
 **state**      | _string_                 | The state of the buildpack; valid states are: `AWAITING_UPLOAD`, `READY`
-**stack**      | _string_                 | The name of the stack that the buildpack will use
+**stack**      | _string_                 | The name of the stack that the buildpack uses
+**lifecycle**  | _string_                 | The version of buildpacks the buildpack uses. `buildpack` indicates [Classic Buildpacks](https://docs.cloudfoundry.org/buildpacks/classic.html). `cnb` indicates [Cloud Native Buildpacks](https://docs.cloudfoundry.org/buildpacks/cnb/)
 **filename**   | _string_                 | The filename of the buildpack
 **position**   | _integer_                | The order in which the buildpacks are checked during buildpack auto-detection
 **enabled**    | _boolean_                | Whether or not the buildpack can be used for staging

lib/cloud_controller/install_buildpacks.rb

@@ -75,7 +75,7 @@ def detected_stack(file, opts)
     def buildpack_zip(package, zipfile)
       return zipfile if zipfile
 
-      job_dir = File.join('/var/vcap/packages', package, '*.zip')
+      job_dir = File.join('/var/vcap/packages', package, '*[.zip|.cnb|.tgz|.tar.gz]')
       Dir[job_dir].first
     end
 

lib/cloud_controller/paging/pagination_options.rb

@@ -12,7 +12,7 @@ class PaginationOptions
     DIRECTION_DEFAULT = 'asc'.freeze
     VALID_DIRECTIONS  = %w[asc desc].freeze
 
-    attr_writer :order_by, :order_direction, :default_order_by
+    attr_writer :order_by, :order_direction, :default_order_by, :secondary_default_order_by
     attr_accessor :page, :per_page
 
     def initialize(params)
@@ -38,6 +38,12 @@ def order_direction
       @order_direction || DIRECTION_DEFAULT
     end
 
+    def secondary_order_by
+      return if @order_by && @order_by.to_s != default_order_by.to_s
+
+      @secondary_default_order_by
+    end
+
     def keys
       %i[page per_page order_by order_direction]
     end

lib/cloud_controller/paging/sequel_paginator.rb

@@ -13,6 +13,8 @@ def get_page(dataset, pagination_options)
       order_type = Sequel.send(order_direction, Sequel.qualify(table_name, order_by))
       dataset = dataset.order(order_type)
 
+      secondary_order_by = pagination_options.secondary_order_by
+      dataset = dataset.order_append(Sequel.send(order_direction, Sequel.qualify(table_name, secondary_order_by))) if secondary_order_by
       dataset = dataset.order_append(Sequel.send(order_direction, Sequel.qualify(table_name, :guid))) if order_by != 'id' && has_guid_column
 
       distinct_opt = dataset.opts[:distinct]