Merge remote-tracking branch 'origin/develop' into feature/s7-discovery

# Conflicts:
#	RELEASE_NOTES

RELEASE_NOTES

@@ -19,6 +19,8 @@ New Features
   them back together. It is now possible to read arrays of
   almost unlimited size.
 - Added auto-discovery to the EIP, KNXNet/IP and S7 Drivers.
+- Added an Optimizer to the Modbus driver, that improves read
+  performance of multi-item read requests by more than 10 times.
 
 Incompatible changes
 --------------------

code-generation/language-c/src/test/resources/integration-test/include/plc4c/driver_test_static.h

@@ -27,8 +27,8 @@
 
 int8_t plc4c_test_read_write_crc_int8();
 uint8_t plc4c_test_read_write_crc_uint8();
-uint8_t plc4c_test_read_write_read_manual_field(plc4c_spi_read_buffer* readBuffer, uint8_t value);
-plc4c_return_code plc4c_test_read_write_write_manual_field(plc4c_spi_write_buffer* writeBuffer, uint8_t value);
+uint8_t plc4c_test_read_write_read_a_manual_field(plc4c_spi_read_buffer* readBuffer, uint8_t value);
+plc4c_return_code plc4c_test_read_write_write_a_manual_field(plc4c_spi_write_buffer* writeBuffer, uint8_t value);
 
 #ifdef __cplusplus
 }

code-generation/language-cs/src/main/resources/templates/cs/data-io-template.cs.ftlh

@@ -99,8 +99,11 @@ namespace org.apache.plc4net.drivers.${protocolName?replace("-", "")}.${outputFl
                         <#if arrayField.loopExpression.contains("curPos")>
                 curPos = readBuffer.getPos() - startPos;
                         </#if>
+                        <#if elementTypeReference.isByteBased()>
+                var ${arrayField.name} = readBuffer.ReadByteArray("", ${helper.toParseExpression(arrayField, helper.intTypeReference, arrayField.loopExpression, parserArguments)});
+                        <#else>
                 <#-- If this is a count array, we can directly initialize an array with the given size -->
-                        <#if field.isCountArrayField()>
+                            <#if field.isCountArrayField()>
                 // Count array
                 List<IPlcValue> ${arrayField.name};
                 {
@@ -111,40 +114,41 @@ namespace org.apache.plc4net.drivers.${protocolName?replace("-", "")}.${outputFl
                     }
                 }
                 <#-- In all other cases do we have to work with a list, that is later converted to an array -->
-                        <#else>
+                            <#else>
                 <#-- For a length array, we read data till the read position of the buffer reaches a given position -->
-                            <#if arrayField.isLengthArrayField()>
+                                <#if arrayField.isLengthArrayField()>
                 // Length array
                 var _${arrayField.name}Length = ${helper.toParseExpression(arrayField, helper.intTypeReference, arrayField.loopExpression,parserArguments)};
                 var ${arrayField.name}EndPos = readBuffer.getPos() + _${arrayField.name}Length;
                 var value = new List<IPlcValue>();
                 while(readBuffer.getPos() < ${arrayField.name}EndPos) {
                     value.Add(
-                <#if elementTypeReference.isSimpleTypeReference()>
+                    <#if elementTypeReference.isSimpleTypeReference()>
                         new ${helper.getPlcValueTypeForTypeReference(elementTypeReference)}(${helper.getReadBufferReadMethodCall(elementTypeReference.asSimpleTypeReference().orElseThrow(), "", arrayField)})
-                <#else>${elementTypeReference.asNonSimpleTypeReference().orElseThrow().name}IO.StaticParse(readBuffer
-                    <#if elementTypeReference.asNonSimpleTypeReference().orElseThrow().params.isPresent()>,
-                        <#list elementTypeReference.asNonSimpleTypeReference().orElseThrow().params.orElseThrow() as parserArgument>
+                    <#else>${elementTypeReference.asNonSimpleTypeReference().orElseThrow().name}IO.StaticParse(readBuffer
+                        <#if elementTypeReference.asNonSimpleTypeReference().orElseThrow().params.isPresent()>,
+                            <#list elementTypeReference.asNonSimpleTypeReference().orElseThrow().params.orElseThrow() as parserArgument>
                                 (${helper.getLanguageTypeNameForTypeReference(helper.getArgumentType(elementTypeReference, parserArgument?index))}) (${helper.toParseExpression(arrayField,elementTypeReference, parserArgument,parserArguments)})
-                            <#sep>, </#sep>
-                        </#list>
-                    </#if>
+                                <#sep>, </#sep>
+                            </#list>
+                        </#if>
                         )
-                </#if>
+                    </#if>
                     );
                 }
                 <#-- A terminated array keeps on reading data as long as the termination expression evaluates to false -->
-                            <#elseif arrayField.isTerminatedArrayField()>
+                                <#elseif arrayField.isTerminatedArrayField()>
                 // Terminated array
                 var ${arrayField.name} = new List<${helper.getLanguageTypeNameForField(arrayField)}>();
                 while(!((boolean) (${helper.toParseExpression(arrayField, helper.boolTypeReference, arrayField.loopExpression,parserArguments)}))) {
                     ${arrayField.name}.Add(<#if elementTypeReference.isSimpleTypeReference()>${helper.getReadBufferReadMethodCall(elementTypeReference.asSimpleTypeReference().orElseThrow(), "", arrayField)}<#else>${elementTypeReference.asNonSimpleTypeReference().orElseThrow().name}IO.StaticParse(readBuffer<#if elementTypeReference.asNonSimpleTypeReference().orElseThrow().params.isPresent()>, <#list elementTypeReference.asNonSimpleTypeReference().orElseThrow().params.orElseThrow() as parserArgument>(${helper.getLanguageTypeNameForTypeReference(helper.getArgumentType(elementTypeReference, parserArgument?index))}) (${helper.toParseExpression(arrayField, elementTypeReference, parserArgument, parserArguments)})<#sep>, </#sep></#list></#if>)</#if>);
 
                 <#-- After parsing, update the current position, but only if it's needed -->
-                                <#if arrayField.loopExpression.contains("curPos")>
+                                    <#if arrayField.loopExpression.contains("curPos")>
                     curPos = readBuffer.getPos() - startPos;
-                                </#if>
+                                    </#if>
                 }
+                                </#if>
                             </#if>
                         </#if>
                         <#if arrayField.name == "value">
@@ -218,8 +222,10 @@ namespace org.apache.plc4net.drivers.${protocolName?replace("-", "")}.${outputFl
                 <#-- In this case we need to wrap each field in a IPlcValue that matches it's natural type -->
                 var _map = new Dictionary<string, IPlcValue>();
                 <#list case.fields as field>
-                    <#if field.isArrayField()>
+                    <#if field.isArrayField() && field.asArrayField().orElseThrow().type.elementTypeReference.isByteBased()>
                         <#assign field=field.asArrayField().orElseThrow()>
+                _map["${field.name}"] = new PlcRawByteArray(${field.name});
+                    <#elseif field.isArrayField()>
                 _map["${field.name}"] = new PlcList(${field.name});
                     <#elseif field.isPropertyField()>
                         <#assign field=field.asPropertyField().orElseThrow()>
@@ -321,7 +327,7 @@ namespace org.apache.plc4net.drivers.${protocolName?replace("-", "")}.${outputFl
                 return new Plc${case.name}(value);
                 </#switch>
             </#if>
-            }<#sep> else </#sep></#list>
+            } </#list>
         <#if !defaultCaseOutput>
             return null;
         </#if>
@@ -422,7 +428,7 @@ namespace org.apache.plc4net.drivers.${protocolName?replace("-", "")}.${outputFl
                 </#switch>
             </#list>
             return writeBuffer;
-        }<#sep> else </#sep></#list>
+        } </#list>
         <#if !defaultCaseOutput>
         return null;
         </#if>

code-generation/language-go/src/main/resources/templates/go/complex-type-template.go.ftlh

@@ -378,6 +378,23 @@ func New${type.name}(
 	) *_${type.name} {
 </@compress>
 
+<#list type.propertyFields as field>
+	<#if !field.isOptionalField()
+		&& !field.isManualField()
+		&& !field.isManualArrayField()
+		&& !field.isArrayField()
+		&& !field.isCountArrayField()
+		&& !field.isLengthArrayField()
+		&& !field.isTerminatedArrayField()
+		&& field.isNamedField()
+		&& field.isTypedField()
+		&& field.asTypedField().orElseThrow().type.isComplexTypeReference()>
+		<#assign fieldName = field.asNamedField().orElseThrow().name>
+if ${fieldName} == nil {
+	panic("${fieldName} of type ${helper.getLanguageTypeNameForField(field)} for ${type.name} must not be nil")
+}
+	</#if>
+</#list>
 <#if type.parentType.isPresent()>
 	_result := &_${type.name}{
 	${type.parentType.orElseThrow().name}Contract: New${type.parentType.orElseThrow().name}(<#list type.parentPropertyFields as parentField>${parentField.name}<#sep>, </#list><#if filteredParentParserArguments?has_content><#if type.parentPropertyFields?has_content>, </#if><#list filteredParentParserArguments as arg>${arg.name}<#sep>, </#sep></#list></#if>),

code-generation/language-go/src/main/resources/templates/go/data-io-template.go.ftlh

@@ -120,6 +120,15 @@ func ${type.name}ParseWithBuffer(ctx context.Context, readBuffer utils.ReadBuffe
                     <#assign arrayElementType = arrayField.type.elementTypeReference>
 
 			// Array Field (${arrayField.name})
+                        <#if arrayElementType.isByteBased()>
+							<#if !field.isCountArrayField() && !field.isLengthArrayField()>
+			return nil, errors.Wrap(_${arrayField.name}Err, "Array fields of type byte only support 'count' and 'length' loop-types.")<@emitImport import="github.com/pkg/errors" />
+							</#if>
+			${arrayField.name}, _${arrayField.name}Err := readBuffer.ReadByteArray("${arrayField.name}", int(${helper.toParseExpression(null, null, arrayField.loopExpression, parserArguments)}))<#if arrayField.loopExpression.contains("CEIL")><@emitImport import="math" /></#if>
+			if _${arrayField.name}Err != nil {
+				return nil, errors.Wrap(_${arrayField.name}Err, "Error parsing '${arrayField.name}' field")<@emitImport import="github.com/pkg/errors" />
+			}
+                        <#else>
 			var ${arrayField.name} []api.PlcValue
 			for i := 0; i < int(${helper.toParseExpression(null, null, arrayField.loopExpression, parserArguments)}); i++ {
 				_item, _itemErr := <#if arrayElementType.isSimpleTypeReference()>${helper.getReadBufferReadMethodCall(arrayField.name, arrayElementType.asSimpleTypeReference().orElseThrow(), arrayField)}<#else>Complex type array in data-io parsing currently not implemented</#if>
@@ -128,6 +137,7 @@ func ${type.name}ParseWithBuffer(ctx context.Context, readBuffer utils.ReadBuffe
 				}
 				${arrayField.name} = append(${arrayField.name}, ${helper.getPlcValueTypeForTypeReference(arrayElementType)}(_item))
 			}
+						</#if>
 					<#if arrayField.name == "value">
 						<#assign valueDefined=true>
 					</#if>

code-generation/language-java/src/test/resources/integration-test/src/main/java/org/apache/plc4x/java/test/readwrite/utils/StaticHelper.java

@@ -71,11 +71,11 @@ public static String parseString(ReadBuffer io) {
     public static void serializeString(WriteBuffer io, String data) {
     }
 
-    public static short readManualAField(ReadBuffer io, short simpleField) {
+    public static short readAManualField(ReadBuffer io, short simpleField) {
         return 0;
     }
 
-    public static void writeManualField(WriteBuffer io, short simpleFlied) {
+    public static void writeAManualField(WriteBuffer io, short simpleFlied) {
 
     }
 

code-generation/language-python/src/main/resources/templates/python/data-io-template.python.ftlh

@@ -76,7 +76,7 @@ class ${type.name}:
                         <#if discriminatorType.isEnumTypeReference()>
                         ${helper.getLanguageTypeNameForTypeReference(discriminatorType)}.${helper.toParseExpression(dataIoTypeDefinition.switchField.orElseThrow(), discriminatorType, discriminatorValueTerm, parserArguments)}
                         <#else>
-                        ${helper.camelCaseToSnakeCase(helper.toParseExpression(dataIoTypeDefinition.switchField.orElseThrow(), discriminatorType, discriminatorValueTerm, parserArguments))}
+                        ${helper.toParseExpression(dataIoTypeDefinition.switchField.orElseThrow(), discriminatorType, discriminatorValueTerm, parserArguments)}
                         </#if>
                         <#sep> and </#sep>
                     </#list>
@@ -131,7 +131,7 @@ class ${type.name}:
                 )
                 </@compress>
 
-            <#-- A terminated array keeps on reading data as long as the termination expression evaluates to false -->
+            <#-- A terminated array keeps on reading data as long as the termination expression evaluates to False -->
                             <#elseif arrayField.isTerminatedArrayField()>
             # Terminated array
             ${arrayField.name}: ${helper.getNonPrimitiveLanguageTypeNameForField(arrayField)} = new LinkedList<>()
@@ -377,8 +377,8 @@ class ${type.name}:
             for val in values.get_list():
                 <#if elementTypeReference.isByteBased()>
                 <@emitImport import="from typing import List" />
-                value: list[byte] = val.get_raw()
-                    write_buffer.write_byte_array("", value)
+                value: ${helper.getLanguageTypeNameForField(arrayField)} = val.get_raw()
+                write_buffer.write_byte_array("", value)
                 <#else>
                 value: ${helper.getLanguageTypeNameForTypeReference(elementTypeReference)} = val.get_${helper.camelCaseToSnakeCase(helper.getLanguageTypeNameForTypeReference(elementTypeReference)?cap_first)}()
                 ${helper.getWriteBufferWriteMethodCall(elementTypeReference.asSimpleTypeReference().orElseThrow(), "(" + arrayField.name + ")", arrayField)}
@@ -387,7 +387,7 @@ class ${type.name}:
 
                     <#if case.name == "BOOL">
             while write_buffer.getPos() < len(write_buffer.get_data()):
-                        write_buffer.write_bit(false)
+                        write_buffer.write_bit(False)
                     </#if>
                     <#break>
                     <#case "const">

code-generation/protocol-test/src/main/resources/protocols/test/test.mspec

@@ -121,7 +121,7 @@
             [simple string 8 abstractStringField]
         ]
     ]
-    [abstract bit oneMoreBit]
+    //[abstract bit oneMoreBit] // TODO: apparently this breaks java
 ]
 
 [type ArrayTypeTest

plc4go/internal/bacnetip/apdu.go → plc4go/.plc4xLicencer.header

@@ -16,5 +16,3 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-
-package bacnetip

plc4go/README.md

@@ -34,11 +34,25 @@ To use plc4go import `"github.com/apache/plc4x/plc4go/pkg/api"` and use the driv
 `plc4xpcapanalyzer` is a small tool to evaluate the plc4x parsing against a pcap and report the success rate.
 
 To install `plc4xpcapanalyzer` run: 
-`go install github.com/apache/plc4x/plc4go/tools/plc4xpcapanalyzer@latest`
+`go install github.com/apache/plc4x-extras/plc4go/tools/plc4xpcapanalyzer@latest`
 
 ### plc4xbrowser
 
 `plc4xbrowser` is a terminal application to run `REPL`-commands against a PLC using plc4x (plc4go)
 
 To install `plc4xbrowser` run: 
-`go install github.com/apache/plc4x/plc4go/tools/plc4xbrowser@latest`
+`go install github.com/apache/plc4x-extras/plc4go/tools/plc4xbrowser@latest`
+
+### plc4xLicencer
+
+`plc4xLicencer` is a generator to add the Apache licence to files
+
+To install `plc4xLicencer` run: 
+`go install github.com/apache/plc4x-extras/plc4go/tools/plc4xLicencer@latest`
+
+### plc4xGenerator
+
+`plc4xGenerator` is a generator to generate Serializable and String methods
+
+To install `plc4xGenerator` run: 
+`go install github.com/apache/plc4x-extras/plc4go/tools/plc4xGenerator@latest`

plc4go/internal/ads/MessageCodec.go

@@ -34,12 +34,12 @@ import (
 	"github.com/apache/plc4x/plc4go/spi/utils"
 )
 
-//go:generate go run ../../tools/plc4xgenerator/gen.go -type=MessageCodec
+//go:generate plc4xGenerator -type=MessageCodec
 type MessageCodec struct {
 	_default.DefaultCodec
 	none bool // TODO: just a empty field to satisfy generator (needs fixing because in this case here we have the delegate)
 
-	log zerolog.Logger `ignore:"true"`
+	log zerolog.Logger
 }
 
 func NewMessageCodec(transportInstance transports.TransportInstance, _options ...options.WithOption) *MessageCodec {

plc4go/internal/ads/model/AdsSubscriptionHandle.go

@@ -31,7 +31,7 @@ import (
 	"github.com/apache/plc4x/plc4go/spi/options"
 )
 
-//go:generate go run ../../../tools/plc4xgenerator/gen.go -type=AdsSubscriptionHandle
+//go:generate plc4xGenerator -type=AdsSubscriptionHandle
 type AdsSubscriptionHandle struct {
 	subscriber spi.PlcSubscriber
 	tagName    string