I ended up adding something which is working here.
Jquery ui widget which toggles height of the parent addmore

It’s a bit crude right now, as I’m just manually adjusting the addmore group height on widget button toggle, probably the field template would need customizing to make this more robust, but it’s a starting point at least.

        /**
         *
         *  @custom.groupwrapper
         *  
         */
        $(function () {
            $.widget("custom.groupWrapper", {

                _create: function () {
                    this.wrapper = $("<span>")
                        .addClass("custom-groupwrapper")
                        .prependTo(this.element);

                    this._createDrawerButton();
                },

                _createDrawerButton: function () {

                    console.log(this.wrapper);
                    var _this = this;

                    $("<a>")
                        .attr("tabIndex", -1)
                        .attr("title", "Collapse")
                        .prependTo(this.wrapper)
                        .button({
                            icons: {
                                primary: "ui-icon-triangle-1-s"
                            },
                            text: false
                        })
                        .removeClass("ui-corner-all")
                        .addClass("ui-corner-right")
                        .on("click", function () {
                            $(_this.wrapper)
                                .closest('.piklist-field-addmore-wrapper-full')
                                .toggleClass('hide');
                        });
                },

                _destroy: function () {
                    this.wrapper.remove();
                    this.element.show();
                }
            });

            //@todo unify combobox autocomplete selector
             $(".piklist-field-addmore-wrapper-full").groupWrapper();
        });

draft CSS:

 .piklist-field-addmore-wrapper-full {
    &.hide {
      height: 100px;
      overflow: hidden;
    }
  }

  .custom-groupwrapper {
    position: absolute;
    right: 0;
    a {
      background: none !important;
      border: none !important;
      box-shadow: none;
    }
  }